vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  44 import org.opendaylight.genius.mdsalutil.NWUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.infrautils.caches.CacheProvider;
  49 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  50 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  52 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  58 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  59 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  60 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  61 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 108 import org.slf4j.Logger;
 109 import org.slf4j.LoggerFactory;
 110
 111 @Singleton
 112 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 113
 114     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 115     private static final short DJC_MAX_RETRIES = 3;
 116
 117     private final DataBroker dataBroker;
 118     private final ManagedNewTransactionRunner txRunner;
 119     private final IBgpManager bgpManager;
 120     private final IFibManager fibManager;
 121     private final IMdsalApiManager mdsalManager;
 122     private final IdManagerService idManager;
 123     private final OdlInterfaceRpcService ifaceMgrRpcService;
 124     private final VpnFootprintService vpnFootprintService;
 125     private final IInterfaceManager interfaceManager;
 126     private final IVpnManager vpnManager;
 127     private final ArpResponderHandler arpResponderHandler;
 128     private final JobCoordinator jobCoordinator;
 129     private final VpnUtil vpnUtil;
 130
 131     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 132
 133     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 134             new ConcurrentHashMap<>();
 135
 136     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 137
 138     @Inject
 139     public VpnInterfaceManager(final DataBroker dataBroker,
 140                                final IBgpManager bgpManager,
 141                                final IdManagerService idManager,
 142                                final IMdsalApiManager mdsalManager,
 143                                final IFibManager fibManager,
 144                                final OdlInterfaceRpcService ifaceMgrRpcService,
 145                                final VpnFootprintService vpnFootprintService,
 146                                final IInterfaceManager interfaceManager,
 147                                final IVpnManager vpnManager,
 148                                final ArpResponderHandler arpResponderHandler,
 149                                final JobCoordinator jobCoordinator,
 150                                final CacheProvider cacheProvider,
 151                                final VpnUtil vpnUtil) {
 152         super(VpnInterface.class, VpnInterfaceManager.class);
 153
 154         this.dataBroker = dataBroker;
 155         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 156         this.bgpManager = bgpManager;
 157         this.idManager = idManager;
 158         this.mdsalManager = mdsalManager;
 159         this.fibManager = fibManager;
 160         this.ifaceMgrRpcService = ifaceMgrRpcService;
 161         this.vpnFootprintService = vpnFootprintService;
 162         this.interfaceManager = interfaceManager;
 163         this.vpnManager = vpnManager;
 164         this.arpResponderHandler = arpResponderHandler;
 165         this.jobCoordinator = jobCoordinator;
 166         this.vpnUtil = vpnUtil;
 167
 168         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 169                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 170                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 171     }
 172
 173     public Runnable isNotifyTaskQueued(String intfName) {
 174         return vpnIntfMap.remove(intfName);
 175     }
 176
 177     @PostConstruct
 178     public void start() {
 179         LOG.info("{} start", getClass().getSimpleName());
 180         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 181     }
 182
 183     @Override
 184     @PreDestroy
 185     public void close() {
 186         super.close();
 187         vpnInstanceOpDataEntryCache.close();
 188     }
 189
 190     @Override
 191     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 192         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 193     }
 194
 195     @Override
 196     protected VpnInterfaceManager getDataTreeChangeListener() {
 197         return VpnInterfaceManager.this;
 198     }
 199
 200     @Override
 201     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 202         LOG.info("add: intfName {} onto vpnName {}",
 203                  vpnInterface.getName(),
 204                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 205         addVpnInterface(identifier, vpnInterface, null, null);
 206     }
 207
 208     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 209                           final VpnInterface vpnInterface, String vpnName) {
 210         synchronized (vpnName.intern()) {
 211             if (isVpnInstanceReady(vpnName)) {
 212                 return true;
 213             }
 214             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 215             return false;
 216         }
 217     }
 218
 219     // TODO Clean up the exception handling
 220     @SuppressWarnings("checkstyle:IllegalCatch")
 221     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 222                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 223         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 224             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 225             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 226         }
 227     }
 228
 229     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 230                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 231         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 232         final String interfaceName = key.getName();
 233
 234         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 235             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 236                   vpnName, vpnInterface.getName());
 237             return;
 238         }
 239         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 240              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 241         List<Adjacency> copyOldAdjs = null;
 242         if (oldAdjs != null) {
 243             copyOldAdjs = new ArrayList<>();
 244             copyOldAdjs.addAll(oldAdjs);
 245         }
 246         List<Adjacency> copyNewAdjs = null;
 247         if (newAdjs != null) {
 248             copyNewAdjs = new ArrayList<>();
 249             copyNewAdjs.addAll(newAdjs);
 250         }
 251         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 252     }
 253
 254     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 255                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 256                     final List<Adjacency> newAdjs,
 257                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 258         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 259         final String interfaceName = key.getName();
 260         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 261         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 262             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 263             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 264             if (interfaceState != null) {
 265                 try {
 266                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 267                     final int ifIndex = interfaceState.getIfIndex();
 268                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 269                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 270                         // (the inventory tx goes in last)
 271                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 272                         ListenableFuture<Void> confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 273                             confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 274                                 operTx -> futures.add(
 275                                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> {
 276                                             LOG.info(
 277                                                     "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 278                                                     + " on dpn {}",
 279                                                     vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 280                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 281                                                     confTx, operTx, invTx, interfaceState, vpnName);
 282                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 283                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 284                                                         + " Update for swapping VPN {} case.", interfaceName, vpnName);
 285                                                 if (newAdjs != null) {
 286                                                     for (Adjacency adj : newAdjs) {
 287                                                         if (oldAdjs.contains(adj)) {
 288                                                             oldAdjs.remove(adj);
 289                                                         } else {
 290                                                             if (!isBgpVpnInternetVpn
 291                                                                     || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 292                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 293                                                                         primaryRd, adj, dpnId, operTx, confTx);
 294                                                             }
 295                                                         }
 296                                                     }
 297                                                 }
 298                                                 for (Adjacency adj : oldAdjs) {
 299                                                     if (!isBgpVpnInternetVpn
 300                                                             || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 301                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 302                                                                 operTx, confTx);
 303                                                     }
 304                                                 }
 305                                             }
 306                                         })))));
 307                         futures.add(confFuture);
 308                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 309                                 MoreExecutors.directExecutor());
 310                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 311                                 + " processed successfully", interfaceName, vpnName, dpnId);
 312                         return futures;
 313                     });
 314                 } catch (NumberFormatException | IllegalStateException e) {
 315                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 316                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 317                                     vpnName, e);
 318                     return;
 319                 }
 320             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 321                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 322                     () -> {
 323                         ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
 324                             createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 325                                     confTx, vpnName);
 326                             LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 327                                     vpnName, vpnInterface.getDpnId());
 328                         });
 329                         ListenableFutures.addErrorLogging(future, LOG,
 330                                 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 331                         return Collections.singletonList(future);
 332                     });
 333             } else {
 334                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 335                         + " is not available", interfaceName, vpnName);
 336             }
 337         } else {
 338             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 339                       + " as vpn is pending delete", interfaceName, vpnName,
 340                     vpnInterface.getDpnId());
 341         }
 342     }
 343
 344     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 345     // see comments below.
 346     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 347     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 348             final int lportTag, boolean isInterfaceUp,
 349             WriteTransaction writeConfigTxn,
 350             WriteTransaction writeOperTxn,
 351             WriteTransaction writeInvTxn,
 352             Interface interfaceState,
 353             final String vpnName) {
 354         final String interfaceName = vpnInterface.getName();
 355         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 356                 vpnName);
 357         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 358         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 359         if (!isInterfaceUp) {
 360             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 361                      interfaceName, dpId, vpnName);
 362             long vpnId = vpnUtil.getVpnId(vpnName);
 363             if (vpnId == VpnConstants.INVALID_ID) {
 364                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 365                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 366                         dpId);
 367                 return;
 368             }
 369
 370             boolean waitForVpnInterfaceOpRemoval = false;
 371             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 372                 String opVpnName = opVpnInterface.getVpnInstanceName();
 373                 String primaryInterfaceIp = null;
 374                 if (opVpnName.equals(vpnName)) {
 375                     // Please check if the primary VRF Entry does not exist for VPNInterface
 376                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 377                     // back to back
 378                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 379                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 380                     if (adjs == null) {
 381                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 382                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 383                                 vpnName);
 384                         return;
 385                     }
 386                     for (Adjacency adj : adjs) {
 387                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 388                             primaryInterfaceIp = adj.getIpAddress();
 389                             break;
 390                         }
 391                     }
 392                     if (primaryInterfaceIp == null) {
 393                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 394                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 395                                 dpId, vpnName);
 396                         return;
 397                     }
 398                     // Get the rd of the vpn instance
 399                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 400                     if (vrf != null) {
 401                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 402                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 403                         return;
 404                     }
 405                     waitForVpnInterfaceOpRemoval = true;
 406                 } else {
 407                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 408                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 409                 }
 410             }
 411             if (!waitForVpnInterfaceOpRemoval) {
 412                 // Add the VPNInterface and quit
 413                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 414                         null/*ipAddressSourceValuePair*/,
 415                         true /* add */);
 416                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 417                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 418                 if (!isBgpVpnInternetVpn) {
 419                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 420                 }
 421                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 422                         dpId, vpnName);
 423                 if (interfaceManager.isExternalInterface(interfaceName)) {
 424                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 425                         NwConstants.ADD_FLOW);
 426                 }
 427                 return;
 428             }
 429
 430             // FIB didn't get a chance yet to clean up this VPNInterface
 431             // Let us give it a chance here !
 432             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 433                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 434             try {
 435                 Runnable notifyTask = new VpnNotifyTask();
 436                 synchronized (notifyTask) {
 437                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 438                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 439                     // published here while holding the lock on it so this path will hit the wait before notify can be
 440                     // invoked.
 441                     vpnIntfMap.put(interfaceName, notifyTask);
 442                     try {
 443                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 444                     } catch (InterruptedException e) {
 445                         // Ignored
 446                     }
 447                 }
 448             } finally {
 449                 vpnIntfMap.remove(interfaceName);
 450             }
 451
 452             if (opVpnInterface != null) {
 453                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 454                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 455                         dpId, vpnName);
 456                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 457                 return;
 458             }
 459             // VPNInterface got removed, proceed with Add
 460             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 461                     interfaceName, dpId, vpnName);
 462             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 463                     null/*ipAddressSourceValuePair*/,
 464                     true /* add */);
 465             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 466                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 467             if (!isBgpVpnInternetVpn) {
 468                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 469             }
 470             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 471                     + " FIB to clean up", interfaceName, dpId, vpnName);
 472             if (interfaceManager.isExternalInterface(interfaceName)) {
 473                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 474                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 475             }
 476         } else {
 477             try {
 478                 // Interface is retained in the DPN, but its Link Up.
 479                 // Advertise prefixes again for this interface to BGP
 480                 InstanceIdentifier<VpnInterface> identifier =
 481                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 482                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 483                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 484                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 485                 // Perform similar operation as interface add event for extraroutes.
 486                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 487                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 488                         LogicalDatastoreType.CONFIGURATION, path);
 489                 if (!optAdjacencies.isPresent()) {
 490                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 491                     return;
 492                 }
 493                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 494                 for (Adjacency adjacency : adjacencies) {
 495                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 496                         continue;
 497                     }
 498                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 499                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 500                         continue;
 501                     }
 502                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 503                             dpId, writeOperTxn, writeConfigTxn);
 504                 }
 505             } catch (ReadFailedException e) {
 506                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 507                         interfaceName, vpnName, primaryRd, dpId);
 508             }
 509         }
 510     }
 511
 512     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 513             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 514         Uuid extNetworkId;
 515         try {
 516             // vpn instance of ext-net interface is the network-id
 517             extNetworkId = new Uuid(vpnName);
 518         } catch (IllegalArgumentException e) {
 519             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 520                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 521             return;
 522         }
 523
 524         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 525         if (routerIds == null || routerIds.isEmpty()) {
 526             LOG.info("processExternalVpnInterface: No router is associated with {}."
 527                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 528                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 529             return;
 530         }
 531
 532         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 533                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 534         for (Uuid routerId : routerIds) {
 535             String routerName = routerId.getValue();
 536             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 537             if (Objects.equals(primarySwitch, dpId)) {
 538                 Routers router = vpnUtil.getExternalRouter(routerName);
 539                 if (router != null) {
 540                     if (addOrRemove == NwConstants.ADD_FLOW) {
 541                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 542                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 543                                 dpId, interfaceName, lportTag, writeInvTxn);
 544                     } else {
 545                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 546                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 547                                 dpId, interfaceName, lportTag);
 548                     }
 549                 } else {
 550                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 551                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 552                             interfaceName, dpId, vpnName);
 553                 }
 554             }
 555         }
 556     }
 557
 558     // TODO Clean up the exception handling
 559     @SuppressWarnings("checkstyle:IllegalCatch")
 560     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 561                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 562                                                   String vpnName, String interfaceName) {
 563         if (rd == null) {
 564             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 565                   interfaceName, dpnId, vpnName);
 566             return;
 567         } else {
 568             if (rd.equals(vpnName)) {
 569                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 570                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 571                 return;
 572             }
 573         }
 574         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 575                 interfaceName, dpnId, vpnName, rd);
 576
 577         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 578         if (nextHopIp == null) {
 579             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 580                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 581                     rd, vpnName);
 582             return;
 583         }
 584
 585         try {
 586             //Read NextHops
 587             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 588             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 589                     LogicalDatastoreType.OPERATIONAL, path);
 590             if (adjacencies.isPresent()) {
 591                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 592                 if (!nextHops.isEmpty()) {
 593                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 594                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 595                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 596                     long l3vni = vpnInstanceOpData.getL3vni();
 597                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 598                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 599                     for (Adjacency nextHop : nextHops) {
 600                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 601                             continue;
 602                         }
 603                         String gatewayMac = null;
 604                         long label = 0;
 605                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 606                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 607                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 608                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 609                                     .get();
 610                         } else {
 611                             label = nextHop.getLabel();
 612                         }
 613                         try {
 614                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 615                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 616                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 617                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 618                                     gatewayMac);
 619                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 620                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 621                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 622                         } catch (Exception e) {
 623                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 624                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 625                                     interfaceName, dpnId, e);
 626                         }
 627                     }
 628                 }
 629             }
 630         } catch (ReadFailedException e) {
 631             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 632                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 633         }
 634     }
 635
 636     // TODO Clean up the exception handling
 637     @SuppressWarnings("checkstyle:IllegalCatch")
 638     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 639                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 640         //Read NextHops
 641         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 642         String rd = vpnUtil.getVpnRd(interfaceName);
 643         if (rd == null) {
 644             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 645                 interfaceName, vpnName);
 646             return;
 647         } else {
 648             if (rd.equals(vpnName)) {
 649                 LOG.info(
 650                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 651                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 652                 return;
 653             }
 654         }
 655         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 656                vpnName, rd);
 657         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 658         try {
 659             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 660                     path);
 661         } catch (ReadFailedException e) {
 662             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 663                     interfaceName, vpnName);
 664         }
 665         if (adjacencies.isPresent()) {
 666             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 667
 668             if (!nextHops.isEmpty()) {
 669                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 670                         nextHops, interfaceName, vpnName, rd);
 671                 for (Adjacency nextHop : nextHops) {
 672                     try {
 673                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 674                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 675                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 676                                     interfaceName, vpnName);
 677                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 678                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 679                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 680                                     interfaceName, vpnName);
 681                         } else {
 682                             // Perform similar operation as interface delete event for extraroutes.
 683                             String allocatedRd = nextHop.getVrfId();
 684                             for (String nh : nextHop.getNextHopIpList()) {
 685                                 deleteExtraRouteFromCurrentAndImportingVpns(
 686                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 687                             }
 688                         }
 689                     } catch (Exception e) {
 690                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 691                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 692                     }
 693                 }
 694             }
 695         }
 696     }
 697
 698     @SuppressWarnings("checkstyle:IllegalCatch")
 699     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 700                                                   String primaryRd, String interfaceName, final long vpnId,
 701                                                   WriteTransaction writeConfigTxn,
 702                                                   WriteTransaction writeOperTxn,
 703                                                   final WriteTransaction writeInvTxn,
 704                                                   Interface interfaceState) {
 705         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 706         // Read NextHops
 707         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 708         Optional<Adjacencies> adjacencies = Optional.absent();
 709         try {
 710             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 711                     path);
 712         } catch (ReadFailedException e) {
 713             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 714                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 715         }
 716         if (!adjacencies.isPresent()) {
 717             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 718                     null/*gwMac*/, writeOperTxn);
 719             return;
 720         }
 721         // Get the rd of the vpn instance
 722         String nextHopIp = null;
 723         try {
 724             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 725         } catch (Exception e) {
 726             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 727                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 728         }
 729         List<String> nhList = new ArrayList<>();
 730         if (nextHopIp != null) {
 731             nhList.add(nextHopIp);
 732             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 733                     interfaceName, dpnId, vpnName, nhList);
 734         }
 735         Optional<String> gwMac = Optional.absent();
 736         String vpnInterfaceSubnetGwMacAddress = null;
 737         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 738         Long l3vni = vpnInstanceOpData.getL3vni();
 739         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 740         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 741         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 742         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 743         List<Adjacency> value = new ArrayList<>();
 744         for (Adjacency nextHop : nextHops) {
 745             String rd = primaryRd;
 746             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 747             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 748                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 749                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 750                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 751                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 752                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 753                 continue;
 754             }
 755             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 756                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 757                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 758                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 759                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 760                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 761                 writeOperTxn.merge(
 762                     LogicalDatastoreType.OPERATIONAL,
 763                     VpnUtil.getPrefixToInterfaceIdentifier(
 764                         vpnUtil.getVpnId(vpnName), prefix),
 765                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 766                             prefixCue), true);
 767                 final Uuid subnetId = nextHop.getSubnetId();
 768
 769                 String gatewayIp = nextHop.getSubnetGatewayIp();
 770                 if (gatewayIp == null) {
 771                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 772                     if (gatewayIpOptional.isPresent()) {
 773                         gatewayIp = gatewayIpOptional.get();
 774                     }
 775                 }
 776
 777                 if (gatewayIp != null) {
 778                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 779                     if (gwMac.isPresent()) {
 780                         // A valid mac-address is available for this subnet-gateway-ip
 781                         // Use this for programming ARP_RESPONDER table here.  And save this
 782                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 783                         // to populate L3_GW_MAC_TABLE there.
 784                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 785                                 gatewayIp, gwMac.get());
 786                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 787                     } else {
 788                         // A valid mac-address is not available for this subnet-gateway-ip
 789                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 790                         // Save this connected-mac-address as gateway-mac-address for the
 791                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 792                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 793                         if (gwMac.isPresent()) {
 794                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 795                                     NwConstants.ADD_FLOW, gwMac.get());
 796                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 797                                     gatewayIp, gwMac.get());
 798                         } else {
 799                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 800                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 801                                 + "gwIp {}",
 802                                 subnetId, interfaceName, vpnName, gatewayIp);
 803                         }
 804                     }
 805                 } else {
 806                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 807                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 808                         subnetId, interfaceName, vpnName);
 809                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 810                 }
 811                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 812                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 813             } else {
 814                 //Extra route adjacency
 815                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 816                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 817                 synchronized (vpnPrefixKey.intern()) {
 818                     java.util.Optional<String> rdToAllocate = vpnUtil
 819                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 820                                     nextHop.getNextHopIpList().get(0), dpnId);
 821                     if (rdToAllocate.isPresent()) {
 822                         rd = rdToAllocate.get();
 823                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 824                             rd, prefix);
 825                     } else {
 826                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 827                         continue;
 828                     }
 829                 }
 830                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 831                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 832                         interfaceName, dpnId);
 833             }
 834             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 835             // can be different from the gateway-mac-address within the VRFEntry as the
 836             // gateway-mac-address is a superset.
 837             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 838                     : RouteOrigin.STATIC;
 839             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 840                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 841                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 842             Adjacency operationalAdjacency = null;
 843             try {
 844                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 845             } catch (NullPointerException e) {
 846                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 847                     input, e.getMessage());
 848                 return;
 849             }
 850             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 851                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 852                         vpnName, l3vni, origin,
 853                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 854             }
 855             value.add(operationalAdjacency);
 856         }
 857
 858         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 859         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 860                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 861
 862         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 863                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 864                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 865
 866         for (Adjacency nextHop : aug.getAdjacency()) {
 867             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 868             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 869                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 870                         : RouteOrigin.STATIC;
 871                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 872                 registeredPopulator.populateFib(input, writeConfigTxn);
 873             }
 874         }
 875     }
 876
 877     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 878                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 879         VpnInterfaceOpDataEntry opInterface =
 880               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 881         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 882             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 883         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 884                 WriteTransaction.CREATE_MISSING_PARENTS);
 885         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 886                 interfaceName, dpnId, vpnName);
 887     }
 888
 889     // TODO Clean up the exception handling
 890     @SuppressWarnings("checkstyle:IllegalCatch")
 891     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 892                                            StateTunnelList stateTunnelList,
 893                                            WriteTransaction writeConfigTxn,
 894                                            WriteTransaction writeOperTxn) {
 895
 896         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 897         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 898         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 899         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 900         if (adjList.isEmpty()) {
 901             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 902                     vpnInterface, srcDpnId);
 903             return;
 904         }
 905         String prefix = null;
 906         long label = 0;
 907         List<Adjacency> value = new ArrayList<>();
 908         boolean isNextHopAddReqd = false;
 909         String vpnName = vpnInterface.getVpnInstanceName();
 910         long vpnId = vpnUtil.getVpnId(vpnName);
 911         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 912         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 913                 vpnInterface.getName(), vpnInterface.getDpnId(),
 914                 vpnInterface.getVpnInstanceName(), adjList);
 915         for (Adjacency adj : adjList) {
 916             String rd = adj.getVrfId();
 917             rd = rd != null ? rd : vpnName;
 918             prefix = adj.getIpAddress();
 919             label = adj.getLabel();
 920             List<String> nhList = Collections.singletonList(srcTepIp);
 921             List<String> nextHopList = adj.getNextHopIpList();
 922             // If TEP is added , update the nexthop of primary adjacency.
 923             // Secondary adj nexthop is already pointing to primary adj IP address.
 924             if (nextHopList != null && !nextHopList.isEmpty()) {
 925                 /* everything right already */
 926             } else {
 927                 isNextHopAddReqd = true;
 928             }
 929
 930             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 931                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 932             } else {
 933                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 934                 if (!vrfEntryOptional.isPresent()) {
 935                     continue;
 936                 }
 937                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 938                 if (!nhList.contains(srcTepIp)) {
 939                     nhList.add(srcTepIp);
 940                     isNextHopAddReqd = true;
 941                 }
 942                 value.add(adj);
 943             }
 944
 945             if (isNextHopAddReqd) {
 946                 updateLabelMapper(label, nhList);
 947                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 948                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 949                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 950                 // Update the VRF entry with nextHop
 951                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 952                         label, true, writeConfigTxn);
 953
 954                 //Get the list of VPN's importing this route(prefix) .
 955                 // Then update the VRF entry with nhList
 956                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 957                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 958                     String vpnRd = vpn.getVrfId();
 959                     if (vpnRd != null) {
 960                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 961                                 srcTepIp, label, true, writeConfigTxn);
 962                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 963                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 964                             vpnInterface.getName(), srcDpnId, vpnName,
 965                             vpn.getVpnInstanceName(), vpnRd);
 966                     }
 967                 }
 968                 // Advertise the prefix to BGP only for external vpn
 969                 // since there is a nexthop change.
 970                 try {
 971                     if (!rd.equalsIgnoreCase(vpnName)) {
 972                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 973                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 974                                 null /*gatewayMacAddress*/);
 975                     }
 976                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 977                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 978                             srcDpnId, vpnName);
 979                 } catch (Exception ex) {
 980                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 981                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 982                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 983                 }
 984             }
 985         }
 986         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 987         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 988                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 989                 .addAugmentation(AdjacenciesOp.class, aug).build();
 990         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 991                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 992         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 993                 WriteTransaction.CREATE_MISSING_PARENTS);
 994         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 995                 vpnInterface.getName(), srcDpnId, vpnName);
 996
 997     }
 998
 999     // TODO Clean up the exception handling
1000     @SuppressWarnings("checkstyle:IllegalCatch")
1001     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1002                                               StateTunnelList stateTunnelList,
1003                                               WriteTransaction writeConfigTxn,
1004                                               WriteTransaction writeOperTxn) {
1005
1006         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1007         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1008         String prefix = null;
1009         long label = 0;
1010         boolean isNextHopRemoveReqd = false;
1011         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1012         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1013         String vpnName = vpnInterface.getVpnInstanceName();
1014         long vpnId = vpnUtil.getVpnId(vpnName);
1015         String primaryRd = vpnUtil.getVpnRd(vpnName);
1016         if (adjList != null) {
1017             List<Adjacency> value = new ArrayList<>();
1018             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1019                     vpnInterface.getName(), vpnInterface.getDpnId(),
1020                     vpnInterface.getVpnInstanceName(), adjList);
1021             for (Adjacency adj : adjList) {
1022                 List<String> nhList = new ArrayList<>();
1023                 String rd = adj.getVrfId();
1024                 rd = rd != null ? rd : vpnName;
1025                 prefix = adj.getIpAddress();
1026                 List<String> nextHopList = adj.getNextHopIpList();
1027                 label = adj.getLabel();
1028                 if (nextHopList != null && !nextHopList.isEmpty()) {
1029                     isNextHopRemoveReqd = true;
1030                 }
1031                 // If TEP is deleted , remove the nexthop from primary adjacency.
1032                 // Secondary adj nexthop will continue to point to primary adj IP address.
1033                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1034                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1035                 } else {
1036                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1037                     if (!vrfEntryOptional.isPresent()) {
1038                         continue;
1039                     }
1040                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1041                     if (nhList.contains(srcTepIp)) {
1042                         nhList.remove(srcTepIp);
1043                         isNextHopRemoveReqd = true;
1044                     }
1045                     value.add(adj);
1046                 }
1047
1048                 if (isNextHopRemoveReqd) {
1049                     updateLabelMapper(label, nhList);
1050                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1051                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1052                             prefix, nhList, vpnName,
1053                             vpnId, rd, vpnInterface.getName());
1054                     // Update the VRF entry with removed nextHop
1055                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1056                             label, false, writeConfigTxn);
1057
1058                     //Get the list of VPN's importing this route(prefix) .
1059                     // Then update the VRF entry with nhList
1060                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1061                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1062                         String vpnRd = vpn.getVrfId();
1063                         if (vpnRd != null) {
1064                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1065                                     srcTepIp, label, false, writeConfigTxn);
1066                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1067                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1068                                     nhList, label, vpnInterface.getName(), srcDpnId,
1069                                     vpnName,
1070                                     vpn.getVpnInstanceName(), vpnRd);
1071                         }
1072                     }
1073
1074                     // Withdraw prefix from BGP only for external vpn.
1075                     try {
1076                         if (!rd.equalsIgnoreCase(vpnName)) {
1077                             bgpManager.withdrawPrefix(rd, prefix);
1078                         }
1079                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1080                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1081                                 vpnInterface.getName(), srcDpnId,
1082                                 vpnName);
1083                     } catch (Exception ex) {
1084                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1085                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1086                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1087                     }
1088                 }
1089             }
1090             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1091             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1092                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1093                     .addAugmentation(AdjacenciesOp.class, aug).build();
1094             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1095                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1096             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1097                     WriteTransaction.CREATE_MISSING_PARENTS);
1098             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1099                          vpnInterface.getName(), srcDpnId, vpnName);
1100         }
1101     }
1102
1103     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1104         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1105
1106         String vpnRd = vpnUtil.getVpnRd(vpnName);
1107         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1108         if (vpnInstanceOpDataEntry == null) {
1109             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1110                     + " to check for vpns exporting the routes", vpnName);
1111             return vpnsToExportRoute;
1112         }
1113
1114         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1115             if (input.getVpnInstanceName() == null) {
1116                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1117                         input.getVrfId());
1118                 return false;
1119             }
1120             return !input.getVpnInstanceName().equals(vpnName);
1121         };
1122
1123         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1124             Iterable<String> commonRTs =
1125                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1126                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1127             return Iterators.size(commonRTs.iterator()) > 0;
1128         };
1129
1130         vpnsToExportRoute =
1131                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1132                         Collectors.toList());
1133         return vpnsToExportRoute;
1134     }
1135
1136     // TODO Clean up the exception handling
1137     @SuppressWarnings("checkstyle:IllegalCatch")
1138     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1139         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1140         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1141             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1142             if (vrfEntries != null) {
1143                 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1144                     for (VrfEntry vrfEntry : vrfEntries) {
1145                         try {
1146                             if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1147                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1148                                 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1149                                                 + " is not a controller managed non intervpn link route. Ignoring.",
1150                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1151                                 continue;
1152                             }
1153                             String prefix = vrfEntry.getDestPrefix();
1154                             String gwMac = vrfEntry.getGatewayMacAddress();
1155                             vrfEntry.getRoutePaths().forEach(routePath -> {
1156                                 String nh = routePath.getNexthopAddress();
1157                                 int label = routePath.getLabel().intValue();
1158                                 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1159                                         vrfEntry.getOrigin()))) {
1160                                     LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1161                                                     + " nexthop {} label {} to vpn {} vpnRd {}",
1162                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1163                                     fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1164                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1165                                             0 /*l3vni*/, gwMac,  vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1166                                             confTx);
1167                                 } else {
1168                                     LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1169                                                     + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1170                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1171                                     SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1172                                     importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1173                                             confTx);
1174                                 }
1175                             });
1176                         } catch (RuntimeException e) {
1177                             LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1178                                             + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1179                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1180                         }
1181                     }
1182                 }), LOG, "Error handing VPN exporting routes");
1183             } else {
1184                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1185                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1186             }
1187         }
1188     }
1189
1190     @Override
1191     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1192         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1193         final String interfaceName = key.getName();
1194         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1195             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1196             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1197         }
1198     }
1199
1200     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1201                                 final VpnInterface vpnInterface, final String vpnName,
1202                                 final String interfaceName) {
1203         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1204             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1205                 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1206                     deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1207                     LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1208                 });
1209                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1210                         vpnInterface.getName(), vpnName);
1211                 return Collections.singletonList(future);
1212             }, DJC_MAX_RETRIES);
1213         } else {
1214             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1215             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1216         }
1217     }
1218
1219     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1220     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1221                                 final VpnInterface vpnInterface, final String vpnName,
1222                                 final String interfaceName, final Interface interfaceState) {
1223         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1224                 vpnName, vpnInterface.getDpnId());
1225         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1226         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1227             () -> {
1228                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1229                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1230                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1231                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1232                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1233                                     interfaceName, vpnName);
1234                             BigInteger dpId = BigInteger.ZERO;
1235                             int ifIndex = 0;
1236                             String gwMacAddress = null;
1237                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1238                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1239                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1240                             try {
1241                                 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1242                                         LogicalDatastoreType.OPERATIONAL, interfaceId);
1243                             } catch (ReadFailedException e) {
1244                                 LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName,
1245                                         vpnName);
1246                                 return;
1247                             }
1248                             if (interfaceState != null) {
1249                                 try {
1250                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1251                                 } catch (NumberFormatException | IllegalStateException e) {
1252                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1253                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1254                                                     + " from vpn interface op data store. ", interfaceName,
1255                                             vpnInterface.getDpnId(), vpnName, e);
1256                                     dpId = BigInteger.ZERO;
1257                                 }
1258                                 ifIndex = interfaceState.getIfIndex();
1259                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1260                             } else {
1261                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1262                                         + " from vpn interface op.", interfaceName);
1263                                 if (optVpnInterface.isPresent()) {
1264                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1265                                     dpId = vpnOpInterface.getDpnId();
1266                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1267                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1268                                 } else {
1269                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1270                                                     + " as interfaceState and vpn interface op is not"
1271                                             + " available", interfaceName, vpnName);
1272                                     return;
1273                                 }
1274                             }
1275                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1276                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1277                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1278                             LOG.info(
1279                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1280                                             + "successfully",
1281                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1282                         })))));
1283                 futures.add(configFuture);
1284                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1285                 return futures;
1286             }, DJC_MAX_RETRIES);
1287     }
1288
1289     protected void processVpnInterfaceDown(BigInteger dpId,
1290                                            String interfaceName,
1291                                            int lportTag,
1292                                            String gwMac,
1293                                            VpnInterfaceOpDataEntry vpnOpInterface,
1294                                            boolean isInterfaceStateDown,
1295                                            WriteTransaction writeConfigTxn,
1296                                            WriteTransaction writeOperTxn,
1297                                            WriteTransaction writeInvTxn) {
1298         if (vpnOpInterface == null) {
1299             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1300                     + " as it is not available in operational data store", interfaceName, dpId);
1301             return;
1302         }
1303         final String vpnName = vpnOpInterface.getVpnInstanceName();
1304         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1305                                                     interfaceName, vpnName);
1306         if (!isInterfaceStateDown) {
1307             final long vpnId = vpnUtil.getVpnId(vpnName);
1308             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName, Boolean.TRUE, null);
1309             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1310             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1311                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1312             if (interfaceManager.isExternalInterface(interfaceName)) {
1313                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1314                         NwConstants.DEL_FLOW);
1315             }
1316             if (!isBgpVpnInternetVpn) {
1317                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1318             }
1319             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1320                     + " successful", interfaceName, dpId, vpnName);
1321         } else {
1322             // Interface is retained in the DPN, but its Link Down.
1323             // Only withdraw the prefixes for this interface from BGP
1324             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1325         }
1326     }
1327
1328     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1329                                           final String vpnName, final long vpnId, String gwMac,
1330                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1331                                           final WriteTransaction writeInvTxn) {
1332         //Read NextHops
1333         try {
1334             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1335                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1336             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1337             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1338                     LogicalDatastoreType.OPERATIONAL, path);
1339             String primaryRd = vpnUtil.getVpnRd(vpnName);
1340             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1341                     interfaceName, dpnId, vpnName, primaryRd);
1342             if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1343                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1344                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1345                         interfaceName, dpnId, vpnName, nextHops);
1346                 for (Adjacency nextHop : nextHops) {
1347                     if (nextHop.isPhysNetworkFunc()) {
1348                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1349                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1350                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1351                                 null/*writeCfgTxn*/);
1352                     } else {
1353                         String rd = nextHop.getVrfId();
1354                         List<String> nhList;
1355                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1356                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1357                         } else {
1358                             // This is a primary adjacency
1359                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1360                                     : Collections.emptyList();
1361                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1362                                     interfaceName, writeInvTxn);
1363                         }
1364                         if (!nhList.isEmpty()) {
1365                             if (rd.equals(vpnName)) {
1366                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1367                                 //remove from FIB directly
1368                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1369                                         interfaceName, dpnId, writeConfigTxn));
1370                             } else {
1371                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1372                                         interfaceName, writeConfigTxn);
1373                             }
1374                         } else {
1375                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1376                                             + " interface {}", nextHop.getIpAddress(), rd,
1377                                     nextHop.getAdjacencyType().toString(), interfaceName);
1378                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1379                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1380                         }
1381                     }
1382                     String ip = nextHop.getIpAddress().split("/")[0];
1383                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1384                     if (vpnVipToPort != null) {
1385                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1386                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1387                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1388                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1389                     }
1390                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1391                     if (vpnPortipToPort != null) {
1392                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1393                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1394                                  + "Interface {} ip {} on dpn {} for vpn {}",
1395                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1396                     }
1397                 }
1398             } else {
1399                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1400                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1401                         + " Removing it.", interfaceName, vpnName, dpnId);
1402                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1403             }
1404         } catch (ReadFailedException e) {
1405             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1406                     interfaceName, dpnId, vpnName);
1407         }
1408     }
1409
1410     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1411                                                                    String interfaceName, BigInteger dpnId,
1412                                                                    WriteTransaction writeConfigTxn) {
1413         return (nh) -> {
1414             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1415                     writeConfigTxn);
1416             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1417                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1418                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1419         };
1420     }
1421
1422     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1423                                            BigInteger dpnId, String rd, String interfaceName,
1424                                            WriteTransaction writeConfigTxn) {
1425         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1426                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1427         nhList.forEach((nh) -> {
1428             //IRT: remove routes from other vpns importing it
1429             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1430                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1431             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1432                 String vpnRd = vpn.getVrfId();
1433                 if (vpnRd != null) {
1434                     fibManager.removeOrUpdateFibEntry(vpnRd,
1435                             nextHop.getIpAddress(), nh, writeConfigTxn);
1436                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1437                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1438                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1439                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1440                 }
1441             }
1442         });
1443     }
1444
1445     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1446                                                  int lportTag, String gwMac, String interfaceName,
1447                                                  WriteTransaction writeInvTxn) {
1448         final Uuid subnetId = nextHop.getSubnetId();
1449         if (nextHop.getSubnetGatewayMacAddress() == null) {
1450             // A valid mac-address was not available for this subnet-gateway-ip
1451             // So a connected-mac-address was used for this subnet and we need
1452             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1453             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1454         }
1455         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1456                 subnetId);
1457     }
1458
1459     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1460                                                   String interfaceName) {
1461         // This is either an extra-route (or) a learned IP via subnet-route
1462         List<String> nhList = null;
1463         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1464         if (nextHopIp == null || nextHopIp.isEmpty()) {
1465             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1466                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1467                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1468                     vpnName);
1469             nhList = Collections.emptyList();
1470         } else {
1471             nhList = Collections.singletonList(nextHopIp);
1472         }
1473         return nhList;
1474     }
1475
1476     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1477         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1478         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1479         // else use connected interface
1480         if (gwPort != null && gwPort.isSubnetIp()) {
1481             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1482                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1483             return Optional.of(gwPort.getMacAddress());
1484         }
1485         return Optional.absent();
1486     }
1487
1488     @Override
1489     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1490         final VpnInterface update) {
1491         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1492                 update.getDpnId(), original.getVpnInstanceNames(),
1493                 update.getVpnInstanceNames());
1494         final String vpnInterfaceName = update.getName();
1495         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1496         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1497         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1498             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1499         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1500         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1501             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1502
1503         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1504         //handles switching between <internal VPN - external VPN>
1505         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1506             LOG.info("update: handled VPNInterface {} on dpn {} update"
1507                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1508                      original.getName(), dpnId,
1509                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1510                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1511             return;
1512         }
1513         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1514             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1515             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1516             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1517             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1518             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1519                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1520                     // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1521                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1522                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1523                         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(operTx -> {
1524                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1525                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1526                             LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1527                                     update.getName(), newVpnName);
1528                             //handle both addition and removal of adjacencies
1529                             //currently, new adjacency may be an extra route
1530                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1531                             if (!oldAdjs.equals(newAdjs)) {
1532                                 for (Adjacency adj : copyNewAdjs) {
1533                                     if (copyOldAdjs.contains(adj)) {
1534                                         copyOldAdjs.remove(adj);
1535                                     } else {
1536                                         // add new adjacency - right now only extra route will hit this path
1537                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1538                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1539                                                     dpnId, operTx, confTx);
1540                                         }
1541                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1542                                                 + " vpn interface {} on vpn {} dpnId {}",
1543                                                 adj.getIpAddress(), adj.getNextHopIpList(),
1544                                                 adj.getLabel(), adj.getSubnetId(), update.getName(),
1545                                                 newVpnName, dpnId);
1546                                     }
1547                                 }
1548                                 for (Adjacency adj : copyOldAdjs) {
1549                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1550                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1551                                                 && !adj.isPhysNetworkFunc()) {
1552                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1553                                                     operTx, confTx);
1554                                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = operTx.read(
1555                                                     LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier).get();
1556                                             if (optVpnInterface.isPresent()) {
1557                                                 VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get();
1558                                                 long vpnId = vpnUtil.getVpnId(newVpnName);
1559                                                 vpnUtil.removePrefixToInterfaceAdj(adj, vpnId,
1560                                                         vpnInterfaceOpDataEntry, operTx);
1561                                             } else {
1562                                                 LOG.info("update: Vpninterface {} not present in Operational",
1563                                                         vpnInterfaceName);
1564                                             }
1565                                             //remove FIB entry
1566                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1567                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1568                                                     + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1569                                             //remove BGP entry
1570                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1571                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1572                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1573                                             }
1574                                         } else {
1575                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1576                                                     operTx, confTx);
1577                                         }
1578                                     }
1579                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1580                                                     + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1581                                                     .getNextHopIpList(),
1582                                             adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1583                                 }
1584                             }
1585                         }));
1586                     }));
1587                     for (ListenableFuture<Void> future : futures) {
1588                         ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1589                                 update.getName(), update.getVpnInstanceNames());
1590                     }
1591                     return futures;
1592                 });
1593             } else {
1594                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1595                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1596             }
1597         }
1598     }
1599
1600     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1601                                                  VpnInterface original, VpnInterface update) {
1602         boolean isSwap = Boolean.FALSE;
1603         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1604         final String interfaceName = key.getName();
1605         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1606             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1607         List<String> oldVpnListCopy = new ArrayList<>();
1608         oldVpnListCopy.addAll(oldVpnList);
1609         List<String> newVpnList = update.getVpnInstanceNames().stream()
1610             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1611         oldVpnList.removeAll(newVpnList);
1612         newVpnList.removeAll(oldVpnListCopy);
1613         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1614             for (String oldVpnName: oldVpnList) {
1615                 isSwap = Boolean.TRUE;
1616                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1617                         + " running config-driven swap removal", interfaceName, oldVpnName);
1618                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1619                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1620                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1621             }
1622             //Wait for previous interface bindings to be removed
1623             try {
1624                 Thread.sleep(2000);
1625             } catch (InterruptedException e) {
1626                 //Ignore
1627             }
1628             for (String newVpnName: newVpnList) {
1629                 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1630                 isSwap = Boolean.TRUE;
1631                 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1632                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1633                             + "running config-driven swap addition", interfaceName, newVpnName);
1634                     final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1635                     final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1636                             ? origAdjs.getAdjacency() : new ArrayList<>();
1637                     final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1638                     final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1639                             ? updateAdjs.getAdjacency() : new ArrayList<>();
1640
1641                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1642                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1643                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1644                             interfaceName, oldVpnListCopy, newVpnName);
1645                 }
1646             }
1647         }
1648         return isSwap;
1649     }
1650
1651     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1652         try {
1653             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1654             synchronized (label.toString().intern()) {
1655                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1656                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1657                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1658                         LogicalDatastoreType.OPERATIONAL, lriIid);
1659                 if (opResult.isPresent()) {
1660                     LabelRouteInfo labelRouteInfo =
1661                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1662                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1663                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1664                 }
1665             }
1666             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1667                     nextHopIpList);
1668         } catch (ReadFailedException e) {
1669             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1670                     nextHopIpList);
1671         } catch (TransactionCommitFailedException e) {
1672             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1673                     nextHopIpList);
1674         }
1675     }
1676
1677     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1678         SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1679
1680         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1681         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1682                 .addAugmentation(SubnetRoute.class, route).build();
1683         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1684         InstanceIdentifierBuilder<VrfTables> idBuilder =
1685             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1686         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1687         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1688         if (writeConfigTxn != null) {
1689             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1690         } else {
1691             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1692         }
1693         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1694                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1695     }
1696
1697     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1698                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1699                                            WriteTransaction writeConfigTxn) {
1700         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1701         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1702         try {
1703             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1704                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1705             if (optVpnInterface.isPresent()) {
1706                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1707                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1708                 String vpnName = currVpnIntf.getVpnInstanceName();
1709                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1710                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1711                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1712                         LogicalDatastoreType.OPERATIONAL, adjPath);
1713                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1714                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1715                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1716                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1717                 List<Adjacency> adjacencies;
1718                 if (optAdjacencies.isPresent()) {
1719                     adjacencies = optAdjacencies.get().getAdjacency();
1720                 } else {
1721                     // This code will be hit in case of first PNF adjacency
1722                     adjacencies = new ArrayList<>();
1723                 }
1724                 long vpnId = vpnUtil.getVpnId(vpnName);
1725                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1726                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1727                 Adjacency operationalAdjacency = null;
1728                 //Handling dual stack neutron port primary adjacency
1729                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1730                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1731                             currVpnIntf.getName(), vpnName);
1732                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1733                             currVpnIntf.getName());
1734                     if (interfaceState != null) {
1735                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1736                                 currVpnIntf.getName(),
1737                                 vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1738                     }
1739                 }
1740                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1741                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1742                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1743                             : RouteOrigin.STATIC;
1744                     String nh = adj.getNextHopIpList().get(0);
1745                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1746                     synchronized (vpnPrefixKey.intern()) {
1747                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1748                                 vpnId, null, prefix, vpnName, nh, dpnId);
1749                         if (rdToAllocate.isPresent()) {
1750                             input.setRd(rdToAllocate.get());
1751                             operationalAdjacency = populator.createOperationalAdjacency(input);
1752                             int label = operationalAdjacency.getLabel().intValue();
1753                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1754                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1755                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1756                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1757                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1758                                     vpnName, label, currVpnIntf.getName(), dpnId);
1759                         } else {
1760                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1761                                     vpnName, prefix);
1762                             return;
1763                         }
1764                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1765                         // Keeping the MPLS check for now.
1766                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1767                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1768                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1769                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1770                             vpnsToImportRoute.forEach(vpn -> {
1771                                 if (vpn.getVrfId() != null) {
1772                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1773                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1774                                             .ifPresent(
1775                                                 rds -> vpnManager.addExtraRoute(
1776                                                         vpnUtil.getVpnName(vpn.getVpnId()),
1777                                                         adj.getIpAddress(), nh, rds,
1778                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1779                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1780                                                         opAdjacency, encapType, writeConfigTxn));
1781                                 }
1782                             });
1783                         }
1784                     }
1785                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1786                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1787                             currVpnIntf.getName(), vpnName);
1788
1789                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1790
1791                     writeOperTxn.merge(
1792                             LogicalDatastoreType.OPERATIONAL,
1793                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1794                                     prefix), VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(),
1795                                     prefix, adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1796
1797                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1798                             adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1799                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1800
1801                     input.setRd(adj.getVrfId());
1802                 }
1803                 if (operationalAdjacency == null) {
1804                     operationalAdjacency = populator.createOperationalAdjacency(input);
1805                 }
1806                 adjacencies.add(operationalAdjacency);
1807                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1808                 VpnInterfaceOpDataEntry newVpnIntf =
1809                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1810                                 aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1811                                 currVpnIntf.getGatewayMacAddress());
1812
1813                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1814             }
1815         } catch (ReadFailedException e) {
1816             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1817                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1818         }
1819     }
1820
1821     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1822         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
1823         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1824     }
1825
1826     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1827             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1828         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1829         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1830         try {
1831             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1832                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1833             if (optVpnInterface.isPresent()) {
1834                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1835                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1836                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1837                         LogicalDatastoreType.OPERATIONAL, path);
1838                 if (optAdjacencies.isPresent()) {
1839                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1840
1841                     if (!adjacencies.isEmpty()) {
1842                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1843                         Iterator<Adjacency> adjIt = adjacencies.iterator();
1844                         while (adjIt.hasNext()) {
1845                             Adjacency adjElem = adjIt.next();
1846                             if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1847                                 String rd = adjElem.getVrfId();
1848                                 if (adj.getNextHopIpList() != null) {
1849                                     for (String nh : adj.getNextHopIpList()) {
1850                                         deleteExtraRouteFromCurrentAndImportingVpns(
1851                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1852                                                 currVpnIntf.getName(), writeConfigTxn);
1853                                     }
1854                                 } else if (adj.isPhysNetworkFunc()) {
1855                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1856                                             adj.getIpAddress(), adj.getSubnetId());
1857                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1858                                             writeConfigTxn);
1859                                 }
1860                                 break;
1861                             }
1862
1863                         }
1864                     }
1865                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1866                             dpnId, adj.getVrfId());
1867                 } else {
1868                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1869                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1870                 }
1871             }
1872         } catch (ReadFailedException e) {
1873             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1874                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1875         }
1876     }
1877
1878     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1879         String rd, String intfName, WriteTransaction writeConfigTxn) {
1880         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1881         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1882         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1883             String vpnRd = vpn.getVrfId();
1884             if (vpnRd != null) {
1885                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1886             }
1887         }
1888     }
1889
1890     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1891         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1892             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1893             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1894     }
1895
1896     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1897         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1898             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1899     }
1900
1901     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1902                                                     WriteTransaction writeConfigTxn, String vpnName) {
1903         if (vpnInterface == null) {
1904             return;
1905         }
1906         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
1907         if (adjs == null) {
1908             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1909                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1910             return;
1911         }
1912         for (Adjacency adj : adjs) {
1913             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1914                 String primaryInterfaceIp = adj.getIpAddress();
1915                 String macAddress = adj.getMacAddress();
1916                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1917
1918                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
1919                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1920
1921                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1922                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1923                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1924                         routerInt, label, writeConfigTxn);
1925                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1926                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1927                         macAddress);
1928                 return;
1929             }
1930         }
1931         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1932                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1933                 primaryRd, vpnName);
1934     }
1935
1936     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1937                                                     WriteTransaction writeConfigTxn, String vpnName) {
1938         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
1939         String rd = vpnUtil.getVpnRd(vpnName);
1940         if (adjs != null) {
1941             List<Adjacency> adjsList = adjs.getAdjacency();
1942             for (Adjacency adj : adjsList) {
1943                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1944                     String primaryInterfaceIp = adj.getIpAddress();
1945                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1946                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1947                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1948                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1949                     return;
1950                 }
1951             }
1952         } else {
1953             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1954                     vpnInterface.getName(), rd);
1955         }
1956     }
1957
1958     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1959         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1960             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1961                   vpnName, intefaceData.vpnInterface.getName());
1962             return;
1963         }
1964         final VpnInterfaceKey key = intefaceData.identifier
1965                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1966         final String interfaceName = key.getName();
1967         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1968                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1969         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1970                   intefaceData.identifier, vpnName);
1971         return;
1972     }
1973
1974     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1975                                               VpnInterface vpnInterface, String vpnName) {
1976         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
1977                .get(vpnName);
1978         if (vpnInterfaces == null) {
1979             vpnInterfaces = new ConcurrentLinkedQueue<>();
1980         }
1981         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
1982         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
1983         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
1984                 vpnInterface.getName(), vpnName);
1985     }
1986
1987     public boolean isVpnInstanceReady(String vpnInstanceName) {
1988         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
1989         if (vpnRd == null) {
1990             return false;
1991         }
1992         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1993
1994         return vpnInstanceOpDataEntry != null;
1995     }
1996
1997     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
1998         synchronized (vpnInstanceName.intern()) {
1999             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2000                     unprocessedVpnInterfaces.get(vpnInstanceName);
2001             if (vpnInterfaces != null) {
2002                 while (!vpnInterfaces.isEmpty()) {
2003                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2004                     if (hasVpnInstanceCreatedSuccessfully) {
2005                         processSavedInterface(savedInterface, vpnInstanceName);
2006                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2007                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2008                     } else {
2009                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2010                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2011                     }
2012                 }
2013             } else {
2014                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2015             }
2016         }
2017     }
2018
2019     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2020             VpnInterface vpnInterface) {
2021         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2022             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2023                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2024             if (vpnInterfaces != null) {
2025                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2026                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2027                             + "unprocessed list", vpnInterface.getName(),
2028                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2029                 }
2030             } else {
2031                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2032                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2033             }
2034         }
2035     }
2036
2037     public void vpnInstanceIsReady(String vpnInstanceName) {
2038         processSavedInterfaces(vpnInstanceName, true);
2039     }
2040
2041     public void vpnInstanceFailed(String vpnInstanceName) {
2042         processSavedInterfaces(vpnInstanceName, false);
2043     }
2044
2045     private static class UnprocessedVpnInterfaceData {
2046         InstanceIdentifier<VpnInterface> identifier;
2047         VpnInterface vpnInterface;
2048
2049         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2050             this.identifier = identifier;
2051             this.vpnInterface = vpnInterface;
2052         }
2053
2054         @Override
2055         public int hashCode() {
2056             final int prime = 31;
2057             int result = 1;
2058             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2059             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2060             return result;
2061         }
2062
2063         @Override
2064         public boolean equals(Object obj) {
2065             if (this == obj) {
2066                 return true;
2067             }
2068             if (obj == null) {
2069                 return false;
2070             }
2071             if (getClass() != obj.getClass()) {
2072                 return false;
2073             }
2074             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2075             if (identifier == null) {
2076                 if (other.identifier != null) {
2077                     return false;
2078                 }
2079             } else if (!identifier.equals(other.identifier)) {
2080                 return false;
2081             }
2082             if (vpnInterface == null) {
2083                 if (other.vpnInterface != null) {
2084                     return false;
2085                 }
2086             } else if (!vpnInterface.equals(other.vpnInterface)) {
2087                 return false;
2088             }
2089             return true;
2090         }
2091     }
2092
2093     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2094         String primaryRd = vpnUtil.getVpnRd(vpnName);
2095         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2096         if (vpnInstanceOpData == null) {
2097             return;
2098         }
2099         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2100         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2101             return;
2102         }
2103         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2104         vpnToDpnLists.forEach(vpnToDpnList -> {
2105             if (vpnToDpnList.getVpnInterfaces() == null) {
2106                 return;
2107             }
2108             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2109                 try {
2110                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2111                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2112                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2113                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2114                     if (!vpnInterfaceOptional.isPresent()) {
2115                         return;
2116                     }
2117                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2118                             vpnInterface.getInterfaceName());
2119                     if (configVpnAdjacencies == null) {
2120                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2121                         return;
2122                     }
2123                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2124                             .augmentation(AdjacenciesOp.class).getAdjacency();
2125                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2126                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2127                     // in operational DS. These unprocessed adjacencies will be handled below.
2128                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2129                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2130                     configVpnAdjacencies.stream()
2131                         .filter(adjacency -> operationVpnAdjacencies.stream()
2132                                 .noneMatch(operationalAdjacency ->
2133                                         operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2134                         .forEach(adjacency -> {
2135                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2136                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2137                                 () -> {
2138                                     // TODO Deal with sequencing — the config tx must only submitted
2139                                     // if the oper tx goes in
2140                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2141                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2142                                         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
2143                                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
2144                                                 confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2145                                                     primaryRd, adjacency, vpnInterfaceOptional.get()
2146                                                         .getDpnId(), confTx, operTx)))));
2147                                         return futures;
2148                                     } else {
2149                                         return Collections.emptyList();
2150                                     }
2151                                 });
2152                         });
2153                 } catch (ReadFailedException e) {
2154                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2155                             vpnName, primaryRd);
2156                 }
2157             });
2158         });
2159     }
2160
2161     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2162         private final String interfaceName;
2163         private final boolean add;
2164         private final String txnDestination;
2165
2166         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2167             this.interfaceName = interfaceName;
2168             this.add = add;
2169             this.txnDestination = transactionDest;
2170         }
2171
2172         @Override
2173         public void onSuccess(Void voidObj) {
2174             if (add) {
2175                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2176                         interfaceName, txnDestination);
2177             } else {
2178                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2179             }
2180         }
2181
2182         @Override
2183         public void onFailure(Throwable throwable) {
2184             if (add) {
2185                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2186                         interfaceName, txnDestination, throwable);
2187             } else {
2188                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2189                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2190             }
2191         }
2192     }
2193 }