vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  44 import org.opendaylight.genius.mdsalutil.NWUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.infrautils.caches.CacheProvider;
  49 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  50 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  52 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  58 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  59 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  60 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  61 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 108 import org.slf4j.Logger;
 109 import org.slf4j.LoggerFactory;
 110
 111 @Singleton
 112 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 113
 114     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 115     private static final short DJC_MAX_RETRIES = 3;
 116
 117     private final DataBroker dataBroker;
 118     private final ManagedNewTransactionRunner txRunner;
 119     private final IBgpManager bgpManager;
 120     private final IFibManager fibManager;
 121     private final IMdsalApiManager mdsalManager;
 122     private final IdManagerService idManager;
 123     private final OdlInterfaceRpcService ifaceMgrRpcService;
 124     private final VpnFootprintService vpnFootprintService;
 125     private final IInterfaceManager interfaceManager;
 126     private final IVpnManager vpnManager;
 127     private final ArpResponderHandler arpResponderHandler;
 128     private final JobCoordinator jobCoordinator;
 129     private final VpnUtil vpnUtil;
 130
 131     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 132
 133     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 134             new ConcurrentHashMap<>();
 135
 136     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 137
 138     @Inject
 139     public VpnInterfaceManager(final DataBroker dataBroker,
 140                                final IBgpManager bgpManager,
 141                                final IdManagerService idManager,
 142                                final IMdsalApiManager mdsalManager,
 143                                final IFibManager fibManager,
 144                                final OdlInterfaceRpcService ifaceMgrRpcService,
 145                                final VpnFootprintService vpnFootprintService,
 146                                final IInterfaceManager interfaceManager,
 147                                final IVpnManager vpnManager,
 148                                final ArpResponderHandler arpResponderHandler,
 149                                final JobCoordinator jobCoordinator,
 150                                final CacheProvider cacheProvider,
 151                                final VpnUtil vpnUtil) {
 152         super(VpnInterface.class, VpnInterfaceManager.class);
 153
 154         this.dataBroker = dataBroker;
 155         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 156         this.bgpManager = bgpManager;
 157         this.idManager = idManager;
 158         this.mdsalManager = mdsalManager;
 159         this.fibManager = fibManager;
 160         this.ifaceMgrRpcService = ifaceMgrRpcService;
 161         this.vpnFootprintService = vpnFootprintService;
 162         this.interfaceManager = interfaceManager;
 163         this.vpnManager = vpnManager;
 164         this.arpResponderHandler = arpResponderHandler;
 165         this.jobCoordinator = jobCoordinator;
 166         this.vpnUtil = vpnUtil;
 167
 168         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 169                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 170                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 171     }
 172
 173     public Runnable isNotifyTaskQueued(String intfName) {
 174         return vpnIntfMap.remove(intfName);
 175     }
 176
 177     @PostConstruct
 178     public void start() {
 179         LOG.info("{} start", getClass().getSimpleName());
 180         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 181     }
 182
 183     @Override
 184     @PreDestroy
 185     public void close() {
 186         super.close();
 187         vpnInstanceOpDataEntryCache.close();
 188     }
 189
 190     @Override
 191     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 192         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 193     }
 194
 195     @Override
 196     protected VpnInterfaceManager getDataTreeChangeListener() {
 197         return VpnInterfaceManager.this;
 198     }
 199
 200     @Override
 201     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 202         LOG.info("add: intfName {} onto vpnName {}",
 203                  vpnInterface.getName(),
 204                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 205         addVpnInterface(identifier, vpnInterface, null, null);
 206     }
 207
 208     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 209                           final VpnInterface vpnInterface, String vpnName) {
 210         synchronized (vpnName.intern()) {
 211             if (isVpnInstanceReady(vpnName)) {
 212                 return true;
 213             }
 214             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 215             return false;
 216         }
 217     }
 218
 219     // TODO Clean up the exception handling
 220     @SuppressWarnings("checkstyle:IllegalCatch")
 221     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 222                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 223         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 224             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 225             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 226         }
 227     }
 228
 229     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 230                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 231         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 232         final String interfaceName = key.getName();
 233
 234         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 235             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 236                   vpnName, vpnInterface.getName());
 237             return;
 238         }
 239         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 240              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 241         List<Adjacency> copyOldAdjs = null;
 242         if (oldAdjs != null) {
 243             copyOldAdjs = new ArrayList<>();
 244             copyOldAdjs.addAll(oldAdjs);
 245         }
 246         List<Adjacency> copyNewAdjs = null;
 247         if (newAdjs != null) {
 248             copyNewAdjs = new ArrayList<>();
 249             copyNewAdjs.addAll(newAdjs);
 250         }
 251         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 252     }
 253
 254     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 255                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 256                     final List<Adjacency> newAdjs,
 257                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 258         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 259         final String interfaceName = key.getName();
 260         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 261         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 262             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 263             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 264             if (interfaceState != null) {
 265                 try {
 266                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 267                     final int ifIndex = interfaceState.getIfIndex();
 268                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 269                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 270                         // (the inventory tx goes in last)
 271                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 272                         ListenableFuture<Void> confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 273                             confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 274                                 operTx -> futures.add(
 275                                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> {
 276                                             LOG.info(
 277                                                     "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 278                                                     + " on dpn {}",
 279                                                     vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 280                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 281                                                     confTx, operTx, invTx, interfaceState, vpnName);
 282                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 283                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 284                                                         + " Update for swapping VPN {} case.", interfaceName, vpnName);
 285                                                 if (newAdjs != null) {
 286                                                     for (Adjacency adj : newAdjs) {
 287                                                         if (oldAdjs.contains(adj)) {
 288                                                             oldAdjs.remove(adj);
 289                                                         } else {
 290                                                             if (!isBgpVpnInternetVpn
 291                                                                     || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 292                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 293                                                                         primaryRd, adj, dpnId, operTx, confTx);
 294                                                             }
 295                                                         }
 296                                                     }
 297                                                 }
 298                                                 for (Adjacency adj : oldAdjs) {
 299                                                     if (!isBgpVpnInternetVpn
 300                                                             || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 301                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 302                                                                 operTx, confTx);
 303                                                     }
 304                                                 }
 305                                             }
 306                                         })))));
 307                         futures.add(confFuture);
 308                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 309                                 MoreExecutors.directExecutor());
 310                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 311                                 + " processed successfully", interfaceName, vpnName, dpnId);
 312                         return futures;
 313                     });
 314                 } catch (NumberFormatException | IllegalStateException e) {
 315                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 316                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 317                                     vpnName, e);
 318                     return;
 319                 }
 320             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 321                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 322                     () -> {
 323                         ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
 324                             createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 325                                     confTx, vpnName);
 326                             LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 327                                     vpnName, vpnInterface.getDpnId());
 328                         });
 329                         ListenableFutures.addErrorLogging(future, LOG,
 330                                 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 331                         return Collections.singletonList(future);
 332                     });
 333             } else {
 334                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 335                         + " is not available", interfaceName, vpnName);
 336             }
 337         } else {
 338             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 339                       + " as vpn is pending delete", interfaceName, vpnName,
 340                     vpnInterface.getDpnId());
 341         }
 342     }
 343
 344     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 345     // see comments below.
 346     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 347     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 348             final int lportTag, boolean isInterfaceUp,
 349             WriteTransaction writeConfigTxn,
 350             WriteTransaction writeOperTxn,
 351             WriteTransaction writeInvTxn,
 352             Interface interfaceState,
 353             final String vpnName) {
 354         final String interfaceName = vpnInterface.getName();
 355         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 356                 vpnName);
 357         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 358         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 359         if (!isInterfaceUp) {
 360             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 361                      interfaceName, dpId, vpnName);
 362             long vpnId = vpnUtil.getVpnId(vpnName);
 363             if (vpnId == VpnConstants.INVALID_ID) {
 364                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 365                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 366                         dpId);
 367                 return;
 368             }
 369
 370             boolean waitForVpnInterfaceOpRemoval = false;
 371             if (opVpnInterface != null) {
 372                 String opVpnName = opVpnInterface.getVpnInstanceName();
 373                 String primaryInterfaceIp = null;
 374                 if (opVpnName.equals(vpnName)) {
 375                     // Please check if the primary VRF Entry does not exist for VPNInterface
 376                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 377                     // back to back
 378                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 379                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 380                     if (adjs == null) {
 381                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 382                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 383                                 vpnName);
 384                         return;
 385                     }
 386                     for (Adjacency adj : adjs) {
 387                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 388                             primaryInterfaceIp = adj.getIpAddress();
 389                             break;
 390                         }
 391                     }
 392                     if (primaryInterfaceIp == null) {
 393                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 394                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 395                                 dpId, vpnName);
 396                         return;
 397                     }
 398                     // Get the rd of the vpn instance
 399                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 400                     if (vrf != null) {
 401                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 402                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 403                         return;
 404                     }
 405                     waitForVpnInterfaceOpRemoval = true;
 406                 } else {
 407                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 408                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 409                 }
 410             }
 411             if (!waitForVpnInterfaceOpRemoval) {
 412                 // Add the VPNInterface and quit
 413                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 414                         null/*ipAddressSourceValuePair*/,
 415                         true /* add */);
 416                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 417                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 418                 if (!isBgpVpnInternetVpn) {
 419                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 420                 }
 421                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 422                         dpId, vpnName);
 423                 if (interfaceManager.isExternalInterface(interfaceName)) {
 424                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 425                         NwConstants.ADD_FLOW);
 426                 }
 427                 return;
 428             }
 429
 430             // FIB didn't get a chance yet to clean up this VPNInterface
 431             // Let us give it a chance here !
 432             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 433                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 434             try {
 435                 Runnable notifyTask = new VpnNotifyTask();
 436                 synchronized (notifyTask) {
 437                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 438                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 439                     // published here while holding the lock on it so this path will hit the wait before notify can be
 440                     // invoked.
 441                     vpnIntfMap.put(interfaceName, notifyTask);
 442                     try {
 443                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 444                     } catch (InterruptedException e) {
 445                         // Ignored
 446                     }
 447                 }
 448             } finally {
 449                 vpnIntfMap.remove(interfaceName);
 450             }
 451
 452             if (opVpnInterface != null) {
 453                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 454                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 455                         dpId, vpnName);
 456                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 457                 return;
 458             }
 459             // VPNInterface got removed, proceed with Add
 460             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 461                     interfaceName, dpId, vpnName);
 462             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 463                     null/*ipAddressSourceValuePair*/,
 464                     true /* add */);
 465             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 466                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 467             if (!isBgpVpnInternetVpn) {
 468                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 469             }
 470             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 471                     + " FIB to clean up", interfaceName, dpId, vpnName);
 472             if (interfaceManager.isExternalInterface(interfaceName)) {
 473                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 474                                lportTag, NwConstants.ADD_FLOW);
 475             }
 476         } else {
 477             try {
 478                 // Interface is retained in the DPN, but its Link Up.
 479                 // Advertise prefixes again for this interface to BGP
 480                 InstanceIdentifier<VpnInterface> identifier =
 481                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 482                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 483                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 484                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 485                 // Perform similar operation as interface add event for extraroutes.
 486                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 487                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 488                         LogicalDatastoreType.CONFIGURATION, path);
 489                 if (!optAdjacencies.isPresent()) {
 490                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 491                     return;
 492                 }
 493                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 494                 for (Adjacency adjacency : adjacencies) {
 495                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 496                         continue;
 497                     }
 498                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 499                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 500                         continue;
 501                     }
 502                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 503                             dpId, writeOperTxn, writeConfigTxn);
 504                 }
 505             } catch (ReadFailedException e) {
 506                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 507                         interfaceName, vpnName, primaryRd, dpId);
 508             }
 509         }
 510     }
 511
 512     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 513         int lportTag, int addOrRemove) {
 514         Uuid extNetworkId;
 515         try {
 516             // vpn instance of ext-net interface is the network-id
 517             extNetworkId = new Uuid(vpnName);
 518         } catch (IllegalArgumentException e) {
 519             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 520                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 521             return;
 522         }
 523
 524         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 525         if (routerIds == null || routerIds.isEmpty()) {
 526             LOG.info("processExternalVpnInterface: No router is associated with {}."
 527                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 528                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 529             return;
 530         }
 531
 532         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 533                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 534         for (Uuid routerId : routerIds) {
 535             String routerName = routerId.getValue();
 536             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 537             if (Objects.equals(primarySwitch, dpId)) {
 538                 Routers router = vpnUtil.getExternalRouter(routerName);
 539                 if (router != null) {
 540                     if (addOrRemove == NwConstants.ADD_FLOW) {
 541                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 542                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 543                                 dpId, interfaceName, lportTag);
 544                     } else {
 545                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 546                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 547                                 dpId, interfaceName, lportTag);
 548                     }
 549                 } else {
 550                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 551                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 552                             interfaceName, dpId, vpnName);
 553                 }
 554             }
 555         }
 556     }
 557
 558     // TODO Clean up the exception handling
 559     @SuppressWarnings("checkstyle:IllegalCatch")
 560     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 561                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 562                                                   String vpnName, String interfaceName) {
 563         if (rd == null) {
 564             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 565                   interfaceName, dpnId, vpnName);
 566             return;
 567         } else {
 568             if (rd.equals(vpnName)) {
 569                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 570                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 571                 return;
 572             }
 573         }
 574         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 575                 interfaceName, dpnId, vpnName, rd);
 576
 577         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 578         if (nextHopIp == null) {
 579             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 580                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 581                     rd, vpnName);
 582             return;
 583         }
 584
 585         try {
 586             //Read NextHops
 587             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 588             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 589                     LogicalDatastoreType.OPERATIONAL, path);
 590             if (adjacencies.isPresent()) {
 591                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 592                 if (!nextHops.isEmpty()) {
 593                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 594                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 595                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 596                     long l3vni = vpnInstanceOpData.getL3vni();
 597                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 598                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 599                     for (Adjacency nextHop : nextHops) {
 600                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 601                             continue;
 602                         }
 603                         String gatewayMac = null;
 604                         long label = 0;
 605                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 606                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 607                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 608                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 609                                     .get();
 610                         } else {
 611                             label = nextHop.getLabel();
 612                         }
 613                         try {
 614                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 615                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 616                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 617                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 618                                     gatewayMac);
 619                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 620                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 621                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 622                         } catch (Exception e) {
 623                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 624                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 625                                     interfaceName, dpnId, e);
 626                         }
 627                     }
 628                 }
 629             }
 630         } catch (ReadFailedException e) {
 631             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 632                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 633         }
 634     }
 635
 636     // TODO Clean up the exception handling
 637     @SuppressWarnings("checkstyle:IllegalCatch")
 638     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 639                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn,
 640                         WriteTransaction writeOperTx) {
 641         //Read NextHops
 642         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 643         String rd = vpnUtil.getVpnRd(interfaceName);
 644         if (rd == null) {
 645             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 646                 interfaceName, vpnName);
 647             return;
 648         } else {
 649             if (rd.equals(vpnName)) {
 650                 LOG.info(
 651                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 652                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 653                 return;
 654             }
 655         }
 656         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 657                vpnName, rd);
 658         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 659         try {
 660             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 661                     path);
 662         } catch (ReadFailedException e) {
 663             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 664                     interfaceName, vpnName);
 665         }
 666         if (adjacencies.isPresent()) {
 667             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 668
 669             if (!nextHops.isEmpty()) {
 670                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 671                         nextHops, interfaceName, vpnName, rd);
 672                 for (Adjacency nextHop : nextHops) {
 673                     try {
 674                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 675                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 676                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 677                                     interfaceName, vpnName);
 678                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 679                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 680                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 681                                     interfaceName, vpnName);
 682                         } else {
 683                             // Perform similar operation as interface delete event for extraroutes.
 684                             String allocatedRd = nextHop.getVrfId();
 685                             for (String nh : nextHop.getNextHopIpList()) {
 686                                 deleteExtraRouteFromCurrentAndImportingVpns(
 687                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 688                                     writeOperTx);
 689                             }
 690                         }
 691                     } catch (Exception e) {
 692                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 693                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 694                     }
 695                 }
 696             }
 697         }
 698     }
 699
 700     @SuppressWarnings("checkstyle:IllegalCatch")
 701     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 702                                                   String primaryRd, String interfaceName, final long vpnId,
 703                                                   WriteTransaction writeConfigTxn,
 704                                                   WriteTransaction writeOperTxn,
 705                                                   final WriteTransaction writeInvTxn,
 706                                                   Interface interfaceState) {
 707         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 708         // Read NextHops
 709         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 710         Optional<Adjacencies> adjacencies = Optional.absent();
 711         try {
 712             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 713                     path);
 714         } catch (ReadFailedException e) {
 715             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 716                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 717         }
 718         if (!adjacencies.isPresent()) {
 719             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 720                     null/*gwMac*/, writeOperTxn);
 721             return;
 722         }
 723         // Get the rd of the vpn instance
 724         String nextHopIp = null;
 725         try {
 726             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 727         } catch (Exception e) {
 728             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 729                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 730         }
 731         List<String> nhList = new ArrayList<>();
 732         if (nextHopIp != null) {
 733             nhList.add(nextHopIp);
 734             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 735                     interfaceName, dpnId, vpnName, nhList);
 736         }
 737         Optional<String> gwMac = Optional.absent();
 738         String vpnInterfaceSubnetGwMacAddress = null;
 739         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 740         Long l3vni = vpnInstanceOpData.getL3vni();
 741         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 742         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 743         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 744         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 745         List<Adjacency> value = new ArrayList<>();
 746         for (Adjacency nextHop : nextHops) {
 747             String rd = primaryRd;
 748             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 749             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 750                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 751                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 752                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 753                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 754                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 755                 continue;
 756             }
 757             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 758                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 759                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 760                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 761                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 762                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 763                 writeOperTxn.merge(
 764                     LogicalDatastoreType.OPERATIONAL,
 765                     VpnUtil.getPrefixToInterfaceIdentifier(
 766                         vpnUtil.getVpnId(vpnName), prefix),
 767                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 768                             prefixCue), true);
 769                 final Uuid subnetId = nextHop.getSubnetId();
 770
 771                 String gatewayIp = nextHop.getSubnetGatewayIp();
 772                 if (gatewayIp == null) {
 773                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 774                     if (gatewayIpOptional.isPresent()) {
 775                         gatewayIp = gatewayIpOptional.get();
 776                     }
 777                 }
 778
 779                 if (gatewayIp != null) {
 780                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 781                     if (gwMac.isPresent()) {
 782                         // A valid mac-address is available for this subnet-gateway-ip
 783                         // Use this for programming ARP_RESPONDER table here.  And save this
 784                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 785                         // to populate L3_GW_MAC_TABLE there.
 786                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 787                                 gatewayIp, gwMac.get());
 788                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 789                     } else {
 790                         // A valid mac-address is not available for this subnet-gateway-ip
 791                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 792                         // Save this connected-mac-address as gateway-mac-address for the
 793                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 794                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 795                         if (gwMac.isPresent()) {
 796                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 797                                     NwConstants.ADD_FLOW, gwMac.get());
 798                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 799                                     gatewayIp, gwMac.get());
 800                         } else {
 801                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 802                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 803                                 + "gwIp {}",
 804                                 subnetId, interfaceName, vpnName, gatewayIp);
 805                         }
 806                     }
 807                 } else {
 808                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 809                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 810                         subnetId, interfaceName, vpnName);
 811                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 812                 }
 813                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 814                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 815             } else {
 816                 //Extra route adjacency
 817                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 818                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 819                 synchronized (vpnPrefixKey.intern()) {
 820                     java.util.Optional<String> rdToAllocate = vpnUtil
 821                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 822                                     nextHop.getNextHopIpList().get(0), dpnId);
 823                     if (rdToAllocate.isPresent()) {
 824                         rd = rdToAllocate.get();
 825                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 826                             rd, prefix);
 827                     } else {
 828                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 829                         continue;
 830                     }
 831                 }
 832                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 833                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 834                         interfaceName, dpnId);
 835             }
 836             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 837             // can be different from the gateway-mac-address within the VRFEntry as the
 838             // gateway-mac-address is a superset.
 839             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 840                     : RouteOrigin.STATIC;
 841             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 842                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 843                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 844             Adjacency operationalAdjacency = null;
 845             try {
 846                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 847             } catch (NullPointerException e) {
 848                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 849                     input, e.getMessage());
 850                 return;
 851             }
 852             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 853                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 854                         vpnName, l3vni, origin,
 855                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 856             }
 857             value.add(operationalAdjacency);
 858         }
 859
 860         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 861         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 862                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 863
 864         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 865                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 866                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 867
 868         for (Adjacency nextHop : aug.getAdjacency()) {
 869             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 870             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 871                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 872                         : RouteOrigin.STATIC;
 873                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 874                 registeredPopulator.populateFib(input, writeConfigTxn);
 875             }
 876         }
 877     }
 878
 879     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 880                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 881         VpnInterfaceOpDataEntry opInterface =
 882               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 883         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 884             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 885         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 886                 WriteTransaction.CREATE_MISSING_PARENTS);
 887         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 888                 interfaceName, dpnId, vpnName);
 889     }
 890
 891     // TODO Clean up the exception handling
 892     @SuppressWarnings("checkstyle:IllegalCatch")
 893     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 894                                            StateTunnelList stateTunnelList,
 895                                            WriteTransaction writeConfigTxn,
 896                                            WriteTransaction writeOperTxn) {
 897
 898         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 899         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 900         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 901         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 902         if (adjList.isEmpty()) {
 903             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 904                     vpnInterface, srcDpnId);
 905             return;
 906         }
 907         String prefix = null;
 908         long label = 0;
 909         List<Adjacency> value = new ArrayList<>();
 910         boolean isNextHopAddReqd = false;
 911         String vpnName = vpnInterface.getVpnInstanceName();
 912         long vpnId = vpnUtil.getVpnId(vpnName);
 913         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 914         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 915                 vpnInterface.getName(), vpnInterface.getDpnId(),
 916                 vpnInterface.getVpnInstanceName(), adjList);
 917         for (Adjacency adj : adjList) {
 918             String rd = adj.getVrfId();
 919             rd = rd != null ? rd : vpnName;
 920             prefix = adj.getIpAddress();
 921             label = adj.getLabel();
 922             List<String> nhList = Collections.singletonList(srcTepIp);
 923             List<String> nextHopList = adj.getNextHopIpList();
 924             // If TEP is added , update the nexthop of primary adjacency.
 925             // Secondary adj nexthop is already pointing to primary adj IP address.
 926             if (nextHopList != null && !nextHopList.isEmpty()) {
 927                 /* everything right already */
 928             } else {
 929                 isNextHopAddReqd = true;
 930             }
 931
 932             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 933                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 934             } else {
 935                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 936                 if (!vrfEntryOptional.isPresent()) {
 937                     continue;
 938                 }
 939                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 940                 if (!nhList.contains(srcTepIp)) {
 941                     nhList.add(srcTepIp);
 942                     isNextHopAddReqd = true;
 943                 }
 944                 value.add(adj);
 945             }
 946
 947             if (isNextHopAddReqd) {
 948                 updateLabelMapper(label, nhList);
 949                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 950                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 951                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 952                 // Update the VRF entry with nextHop
 953                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 954                         label, true, writeConfigTxn);
 955
 956                 //Get the list of VPN's importing this route(prefix) .
 957                 // Then update the VRF entry with nhList
 958                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 959                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 960                     String vpnRd = vpn.getVrfId();
 961                     if (vpnRd != null) {
 962                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 963                                 srcTepIp, label, true, writeConfigTxn);
 964                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 965                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 966                             vpnInterface.getName(), srcDpnId, vpnName,
 967                             vpn.getVpnInstanceName(), vpnRd);
 968                     }
 969                 }
 970                 // Advertise the prefix to BGP only for external vpn
 971                 // since there is a nexthop change.
 972                 try {
 973                     if (!rd.equalsIgnoreCase(vpnName)) {
 974                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 975                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 976                                 null /*gatewayMacAddress*/);
 977                     }
 978                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 979                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 980                             srcDpnId, vpnName);
 981                 } catch (Exception ex) {
 982                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 983                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 984                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 985                 }
 986             }
 987         }
 988         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 989         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 990                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 991                 .addAugmentation(AdjacenciesOp.class, aug).build();
 992         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 993                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 994         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 995                 WriteTransaction.CREATE_MISSING_PARENTS);
 996         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 997                 vpnInterface.getName(), srcDpnId, vpnName);
 998
 999     }
1000
1001     // TODO Clean up the exception handling
1002     @SuppressWarnings("checkstyle:IllegalCatch")
1003     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1004                                               StateTunnelList stateTunnelList,
1005                                               WriteTransaction writeConfigTxn,
1006                                               WriteTransaction writeOperTxn) {
1007
1008         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1009         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1010         String prefix = null;
1011         long label = 0;
1012         boolean isNextHopRemoveReqd = false;
1013         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1014         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1015         String vpnName = vpnInterface.getVpnInstanceName();
1016         long vpnId = vpnUtil.getVpnId(vpnName);
1017         String primaryRd = vpnUtil.getVpnRd(vpnName);
1018         if (adjList != null) {
1019             List<Adjacency> value = new ArrayList<>();
1020             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1021                     vpnInterface.getName(), vpnInterface.getDpnId(),
1022                     vpnInterface.getVpnInstanceName(), adjList);
1023             for (Adjacency adj : adjList) {
1024                 List<String> nhList = new ArrayList<>();
1025                 String rd = adj.getVrfId();
1026                 rd = rd != null ? rd : vpnName;
1027                 prefix = adj.getIpAddress();
1028                 List<String> nextHopList = adj.getNextHopIpList();
1029                 label = adj.getLabel();
1030                 if (nextHopList != null && !nextHopList.isEmpty()) {
1031                     isNextHopRemoveReqd = true;
1032                 }
1033                 // If TEP is deleted , remove the nexthop from primary adjacency.
1034                 // Secondary adj nexthop will continue to point to primary adj IP address.
1035                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1036                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1037                 } else {
1038                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1039                     if (!vrfEntryOptional.isPresent()) {
1040                         continue;
1041                     }
1042                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1043                     if (nhList.contains(srcTepIp)) {
1044                         nhList.remove(srcTepIp);
1045                         isNextHopRemoveReqd = true;
1046                     }
1047                     value.add(adj);
1048                 }
1049
1050                 if (isNextHopRemoveReqd) {
1051                     updateLabelMapper(label, nhList);
1052                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1053                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1054                             prefix, nhList, vpnName,
1055                             vpnId, rd, vpnInterface.getName());
1056                     // Update the VRF entry with removed nextHop
1057                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1058                             label, false, writeConfigTxn);
1059
1060                     //Get the list of VPN's importing this route(prefix) .
1061                     // Then update the VRF entry with nhList
1062                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1063                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1064                         String vpnRd = vpn.getVrfId();
1065                         if (vpnRd != null) {
1066                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1067                                     srcTepIp, label, false, writeConfigTxn);
1068                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1069                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1070                                     nhList, label, vpnInterface.getName(), srcDpnId,
1071                                     vpnName,
1072                                     vpn.getVpnInstanceName(), vpnRd);
1073                         }
1074                     }
1075
1076                     // Withdraw prefix from BGP only for external vpn.
1077                     try {
1078                         if (!rd.equalsIgnoreCase(vpnName)) {
1079                             bgpManager.withdrawPrefix(rd, prefix);
1080                         }
1081                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1082                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1083                                 vpnInterface.getName(), srcDpnId,
1084                                 vpnName);
1085                     } catch (Exception ex) {
1086                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1087                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1088                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1089                     }
1090                 }
1091             }
1092             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1093             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1094                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1095                     .addAugmentation(AdjacenciesOp.class, aug).build();
1096             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1097                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1098             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1099                     WriteTransaction.CREATE_MISSING_PARENTS);
1100             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1101                          vpnInterface.getName(), srcDpnId, vpnName);
1102         }
1103     }
1104
1105     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1106         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1107
1108         String vpnRd = vpnUtil.getVpnRd(vpnName);
1109         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1110         if (vpnInstanceOpDataEntry == null) {
1111             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1112                     + " to check for vpns exporting the routes", vpnName);
1113             return vpnsToExportRoute;
1114         }
1115
1116         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1117             if (input.getVpnInstanceName() == null) {
1118                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1119                         input.getVrfId());
1120                 return false;
1121             }
1122             return !input.getVpnInstanceName().equals(vpnName);
1123         };
1124
1125         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1126             Iterable<String> commonRTs =
1127                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1128                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1129             return Iterators.size(commonRTs.iterator()) > 0;
1130         };
1131
1132         vpnsToExportRoute =
1133                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1134                         Collectors.toList());
1135         return vpnsToExportRoute;
1136     }
1137
1138     // TODO Clean up the exception handling
1139     @SuppressWarnings("checkstyle:IllegalCatch")
1140     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1141         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1142         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1143             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1144             if (vrfEntries != null) {
1145                 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1146                     for (VrfEntry vrfEntry : vrfEntries) {
1147                         try {
1148                             if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1149                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1150                                 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1151                                                 + " is not a controller managed non intervpn link route. Ignoring.",
1152                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1153                                 continue;
1154                             }
1155                             String prefix = vrfEntry.getDestPrefix();
1156                             String gwMac = vrfEntry.getGatewayMacAddress();
1157                             vrfEntry.getRoutePaths().forEach(routePath -> {
1158                                 String nh = routePath.getNexthopAddress();
1159                                 int label = routePath.getLabel().intValue();
1160                                 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1161                                         vrfEntry.getOrigin()))) {
1162                                     LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1163                                                     + " nexthop {} label {} to vpn {} vpnRd {}",
1164                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1165                                     fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1166                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1167                                             0 /*l3vni*/, gwMac,  vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1168                                             confTx);
1169                                 } else {
1170                                     LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1171                                                     + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1172                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1173                                     SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1174                                     importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1175                                             confTx);
1176                                 }
1177                             });
1178                         } catch (RuntimeException e) {
1179                             LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1180                                             + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1181                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1182                         }
1183                     }
1184                 }), LOG, "Error handing VPN exporting routes");
1185             } else {
1186                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1187                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1188             }
1189         }
1190     }
1191
1192     @Override
1193     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1194         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1195         final String interfaceName = key.getName();
1196         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1197             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1198             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1199         }
1200     }
1201
1202     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1203                                 final VpnInterface vpnInterface, final String vpnName,
1204                                 final String interfaceName) {
1205         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1206             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1207                 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1208                     deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1209                     LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1210                 });
1211                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1212                         vpnInterface.getName(), vpnName);
1213                 return Collections.singletonList(future);
1214             }, DJC_MAX_RETRIES);
1215         } else {
1216             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1217             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1218         }
1219     }
1220
1221     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1222     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1223                                 final VpnInterface vpnInterface, final String vpnName,
1224                                 final String interfaceName, final Interface interfaceState) {
1225         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1226                 vpnName, vpnInterface.getDpnId());
1227         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1228         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1229             () -> {
1230                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1231                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1232                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1233                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1234                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1235                                     interfaceName, vpnName);
1236                             BigInteger dpId = BigInteger.ZERO;
1237                             int ifIndex = 0;
1238                             String gwMacAddress = null;
1239                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1240                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1241                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1242                             try {
1243                                 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1244                                         LogicalDatastoreType.OPERATIONAL, interfaceId);
1245                             } catch (ReadFailedException e) {
1246                                 LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName,
1247                                         vpnName);
1248                                 return;
1249                             }
1250                             if (interfaceState != null) {
1251                                 try {
1252                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1253                                 } catch (NumberFormatException | IllegalStateException e) {
1254                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1255                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1256                                                     + " from vpn interface op data store. ", interfaceName,
1257                                             vpnInterface.getDpnId(), vpnName, e);
1258                                     dpId = BigInteger.ZERO;
1259                                 }
1260                                 ifIndex = interfaceState.getIfIndex();
1261                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1262                             } else {
1263                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1264                                         + " from vpn interface op.", interfaceName);
1265                                 if (optVpnInterface.isPresent()) {
1266                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1267                                     dpId = vpnOpInterface.getDpnId();
1268                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1269                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1270                                 } else {
1271                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1272                                                     + " as interfaceState and vpn interface op is not"
1273                                             + " available", interfaceName, vpnName);
1274                                     return;
1275                                 }
1276                             }
1277                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1278                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1279                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1280                             LOG.info(
1281                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1282                                             + "successfully",
1283                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1284                         })))));
1285                 futures.add(configFuture);
1286                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1287                 return futures;
1288             }, DJC_MAX_RETRIES);
1289     }
1290
1291     protected void processVpnInterfaceDown(BigInteger dpId,
1292                                            String interfaceName,
1293                                            int lportTag,
1294                                            String gwMac,
1295                                            VpnInterfaceOpDataEntry vpnOpInterface,
1296                                            boolean isInterfaceStateDown,
1297                                            WriteTransaction writeConfigTxn,
1298                                            WriteTransaction writeOperTxn,
1299                                            WriteTransaction writeInvTxn) {
1300         if (vpnOpInterface == null) {
1301             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1302                     + " as it is not available in operational data store", interfaceName, dpId);
1303             return;
1304         }
1305         final String vpnName = vpnOpInterface.getVpnInstanceName();
1306         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1307                                                     interfaceName, vpnName);
1308         if (!isInterfaceStateDown) {
1309             final long vpnId = vpnUtil.getVpnId(vpnName);
1310             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1311             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1312             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1313                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1314             if (interfaceManager.isExternalInterface(interfaceName)) {
1315                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1316                     NwConstants.DEL_FLOW);
1317             }
1318             if (!isBgpVpnInternetVpn) {
1319                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1320             }
1321             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1322                     + " successful", interfaceName, dpId, vpnName);
1323         } else {
1324             // Interface is retained in the DPN, but its Link Down.
1325             // Only withdraw the prefixes for this interface from BGP
1326             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1327         }
1328     }
1329
1330     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1331                                           final String vpnName, final long vpnId, String gwMac,
1332                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1333                                           final WriteTransaction writeInvTxn) {
1334         //Read NextHops
1335         try {
1336             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1337                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1338             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1339             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1340                     LogicalDatastoreType.OPERATIONAL, path);
1341             String primaryRd = vpnUtil.getVpnRd(vpnName);
1342             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1343                     interfaceName, dpnId, vpnName, primaryRd);
1344             if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1345                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1346                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1347                         interfaceName, dpnId, vpnName, nextHops);
1348                 for (Adjacency nextHop : nextHops) {
1349                     if (nextHop.isPhysNetworkFunc()) {
1350                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1351                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1352                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1353                                 null/*writeCfgTxn*/);
1354                     } else {
1355                         String rd = nextHop.getVrfId();
1356                         List<String> nhList;
1357                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1358                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1359                         } else {
1360                             // This is a primary adjacency
1361                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1362                                     : Collections.emptyList();
1363                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1364                                     interfaceName, writeInvTxn);
1365                         }
1366                         if (!nhList.isEmpty()) {
1367                             if (rd.equals(vpnName)) {
1368                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1369                                 //remove from FIB directly
1370                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1371                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1372                             } else {
1373                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1374                                         interfaceName, writeConfigTxn, writeOperTxn);
1375                             }
1376                         } else {
1377                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1378                                             + " interface {}", nextHop.getIpAddress(), rd,
1379                                     nextHop.getAdjacencyType().toString(), interfaceName);
1380                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1381                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1382                         }
1383                     }
1384                     String ip = nextHop.getIpAddress().split("/")[0];
1385                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1386                     if (vpnVipToPort != null) {
1387                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1388                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1389                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1390                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1391                     }
1392                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1393                     if (vpnPortipToPort != null) {
1394                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1395                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1396                                  + "Interface {} ip {} on dpn {} for vpn {}",
1397                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1398                     }
1399                 }
1400             } else {
1401                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1402                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1403                         + " Removing it.", interfaceName, vpnName, dpnId);
1404                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1405             }
1406         } catch (ReadFailedException e) {
1407             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1408                     interfaceName, dpnId, vpnName);
1409         }
1410     }
1411
1412     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName, String interfaceName,
1413                                                             BigInteger dpnId, WriteTransaction writeConfigTxn,
1414                                                             WriteTransaction writeOperTx) {
1415         return (nh) -> {
1416             String primaryRd = vpnUtil.getVpnRd(vpnName);
1417             String prefix = nextHop.getIpAddress();
1418             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1419             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1420                     nextHop, vpnName, interfaceName, dpnId);
1421             synchronized (vpnNamePrefixKey.intern()) {
1422                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1423                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1424                     //If extra-route is present behind at least one VM, then do not remove or update
1425                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1426                     //prefixtointerface DS
1427                     return;
1428                 }
1429                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1430                         writeConfigTxn);
1431             }
1432             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1433                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1434                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1435         };
1436     }
1437
1438     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1439                                            BigInteger dpnId, String rd, String interfaceName,
1440                                            WriteTransaction writeConfigTxn, WriteTransaction writeOperTx) {
1441         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1442                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1443         nhList.forEach((nh) -> {
1444             //IRT: remove routes from other vpns importing it
1445             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1446                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1447             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1448                 String vpnRd = vpn.getVrfId();
1449                 if (vpnRd != null) {
1450                     fibManager.removeOrUpdateFibEntry(vpnRd,
1451                             nextHop.getIpAddress(), nh, writeConfigTxn);
1452                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1453                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1454                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1455                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1456                 }
1457             }
1458         });
1459     }
1460
1461     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1462                                                  int lportTag, String gwMac, String interfaceName,
1463                                                  WriteTransaction writeInvTxn) {
1464         final Uuid subnetId = nextHop.getSubnetId();
1465         if (nextHop.getSubnetGatewayMacAddress() == null) {
1466             // A valid mac-address was not available for this subnet-gateway-ip
1467             // So a connected-mac-address was used for this subnet and we need
1468             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1469             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1470         }
1471         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1472                 subnetId);
1473     }
1474
1475     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1476                                                   String interfaceName) {
1477         // This is either an extra-route (or) a learned IP via subnet-route
1478         List<String> nhList = null;
1479         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1480         if (nextHopIp == null || nextHopIp.isEmpty()) {
1481             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1482                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1483                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1484                     vpnName);
1485             nhList = Collections.emptyList();
1486         } else {
1487             nhList = Collections.singletonList(nextHopIp);
1488         }
1489         return nhList;
1490     }
1491
1492     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1493         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1494         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1495         // else use connected interface
1496         if (gwPort != null && gwPort.isSubnetIp()) {
1497             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1498                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1499             return Optional.of(gwPort.getMacAddress());
1500         }
1501         return Optional.absent();
1502     }
1503
1504     @Override
1505     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1506         final VpnInterface update) {
1507         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1508                 update.getDpnId(), original.getVpnInstanceNames(),
1509                 update.getVpnInstanceNames());
1510         final String vpnInterfaceName = update.getName();
1511         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1512         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1513         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1514             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1515         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1516         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1517             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1518
1519         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1520         //handles switching between <internal VPN - external VPN>
1521         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1522             LOG.info("update: handled VPNInterface {} on dpn {} update"
1523                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1524                      original.getName(), dpnId,
1525                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1526                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1527             return;
1528         }
1529         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1530             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1531             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1532             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1533             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1534             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1535                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1536                     // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1537                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1538                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1539                         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(operTx -> {
1540                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1541                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1542                             LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1543                                     update.getName(), newVpnName);
1544                             //handle both addition and removal of adjacencies
1545                             //currently, new adjacency may be an extra route
1546                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1547                             if (!oldAdjs.equals(newAdjs)) {
1548                                 for (Adjacency adj : copyNewAdjs) {
1549                                     if (copyOldAdjs.contains(adj)) {
1550                                         copyOldAdjs.remove(adj);
1551                                     } else {
1552                                         // add new adjacency - right now only extra route will hit this path
1553                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1554                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1555                                                     dpnId, operTx, confTx);
1556                                         }
1557                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1558                                                 + " vpn interface {} on vpn {} dpnId {}",
1559                                                 adj.getIpAddress(), adj.getNextHopIpList(),
1560                                                 adj.getLabel(), adj.getSubnetId(), update.getName(),
1561                                                 newVpnName, dpnId);
1562                                     }
1563                                 }
1564                                 for (Adjacency adj : copyOldAdjs) {
1565                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1566                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1567                                                 && !adj.isPhysNetworkFunc()) {
1568                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1569                                                     operTx, confTx);
1570                                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = operTx.read(
1571                                                     LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier).get();
1572                                             if (optVpnInterface.isPresent()) {
1573                                                 VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get();
1574                                                 long vpnId = vpnUtil.getVpnId(newVpnName);
1575                                                 vpnUtil.removePrefixToInterfaceAdj(adj, vpnId,
1576                                                         vpnInterfaceOpDataEntry, operTx);
1577                                             } else {
1578                                                 LOG.info("update: Vpninterface {} not present in Operational",
1579                                                         vpnInterfaceName);
1580                                             }
1581                                             //remove FIB entry
1582                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1583                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1584                                                     + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1585                                             //remove BGP entry
1586                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1587                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1588                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1589                                             }
1590                                         } else {
1591                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1592                                                     operTx, confTx);
1593                                         }
1594                                     }
1595                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1596                                                     + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1597                                                     .getNextHopIpList(),
1598                                             adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1599                                 }
1600                             }
1601                         }));
1602                     }));
1603                     for (ListenableFuture<Void> future : futures) {
1604                         ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1605                                 update.getName(), update.getVpnInstanceNames());
1606                     }
1607                     return futures;
1608                 });
1609             } else {
1610                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1611                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1612             }
1613         }
1614     }
1615
1616     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1617                                                  VpnInterface original, VpnInterface update) {
1618         boolean isSwap = Boolean.FALSE;
1619         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1620         final String interfaceName = key.getName();
1621         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1622             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1623         List<String> oldVpnListCopy = new ArrayList<>();
1624         oldVpnListCopy.addAll(oldVpnList);
1625         List<String> newVpnList = update.getVpnInstanceNames().stream()
1626             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1627         oldVpnList.removeAll(newVpnList);
1628         newVpnList.removeAll(oldVpnListCopy);
1629         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1630             for (String oldVpnName: oldVpnList) {
1631                 isSwap = Boolean.TRUE;
1632                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1633                         + " running config-driven swap removal", interfaceName, oldVpnName);
1634                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1635                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1636                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1637             }
1638             //Wait for previous interface bindings to be removed
1639             try {
1640                 Thread.sleep(2000);
1641             } catch (InterruptedException e) {
1642                 //Ignore
1643             }
1644             for (String newVpnName: newVpnList) {
1645                 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1646                 isSwap = Boolean.TRUE;
1647                 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1648                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1649                             + "running config-driven swap addition", interfaceName, newVpnName);
1650                     final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1651                     final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1652                             ? origAdjs.getAdjacency() : new ArrayList<>();
1653                     final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1654                     final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1655                             ? updateAdjs.getAdjacency() : new ArrayList<>();
1656
1657                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1658                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1659                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1660                             interfaceName, oldVpnListCopy, newVpnName);
1661                 }
1662             }
1663         }
1664         return isSwap;
1665     }
1666
1667     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1668         try {
1669             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1670             synchronized (label.toString().intern()) {
1671                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1672                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1673                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1674                         LogicalDatastoreType.OPERATIONAL, lriIid);
1675                 if (opResult.isPresent()) {
1676                     LabelRouteInfo labelRouteInfo =
1677                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1678                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1679                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1680                 }
1681             }
1682             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1683                     nextHopIpList);
1684         } catch (ReadFailedException e) {
1685             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1686                     nextHopIpList);
1687         } catch (TransactionCommitFailedException e) {
1688             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1689                     nextHopIpList);
1690         }
1691     }
1692
1693     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1694         SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1695
1696         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1697         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1698                 .addAugmentation(SubnetRoute.class, route).build();
1699         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1700         InstanceIdentifierBuilder<VrfTables> idBuilder =
1701             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1702         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1703         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1704         if (writeConfigTxn != null) {
1705             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1706         } else {
1707             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1708         }
1709         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1710                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1711     }
1712
1713     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1714                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1715                                            WriteTransaction writeConfigTxn) {
1716         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1717         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1718         try {
1719             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1720                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1721             if (optVpnInterface.isPresent()) {
1722                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1723                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1724                 String vpnName = currVpnIntf.getVpnInstanceName();
1725                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1726                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1727                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1728                         LogicalDatastoreType.OPERATIONAL, adjPath);
1729                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1730                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1731                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1732                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1733                 List<Adjacency> adjacencies;
1734                 if (optAdjacencies.isPresent()) {
1735                     adjacencies = optAdjacencies.get().getAdjacency();
1736                 } else {
1737                     // This code will be hit in case of first PNF adjacency
1738                     adjacencies = new ArrayList<>();
1739                 }
1740                 long vpnId = vpnUtil.getVpnId(vpnName);
1741                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1742                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1743                 Adjacency operationalAdjacency = null;
1744                 //Handling dual stack neutron port primary adjacency
1745                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1746                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1747                             currVpnIntf.getName(), vpnName);
1748                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1749                             currVpnIntf.getName());
1750                     if (interfaceState != null) {
1751                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1752                                 currVpnIntf.getName(),
1753                                 vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1754                     }
1755                 }
1756                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1757                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1758                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1759                             : RouteOrigin.STATIC;
1760                     String nh = adj.getNextHopIpList().get(0);
1761                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1762                     synchronized (vpnPrefixKey.intern()) {
1763                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1764                                 vpnId, null, prefix, vpnName, nh, dpnId);
1765                         if (rdToAllocate.isPresent()) {
1766                             input.setRd(rdToAllocate.get());
1767                             operationalAdjacency = populator.createOperationalAdjacency(input);
1768                             int label = operationalAdjacency.getLabel().intValue();
1769                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1770                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1771                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1772                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1773                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1774                                     vpnName, label, currVpnIntf.getName(), dpnId);
1775                         } else {
1776                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1777                                     vpnName, prefix);
1778                             return;
1779                         }
1780                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1781                         // Keeping the MPLS check for now.
1782                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1783                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1784                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1785                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1786                             vpnsToImportRoute.forEach(vpn -> {
1787                                 if (vpn.getVrfId() != null) {
1788                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1789                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1790                                             .ifPresent(
1791                                                 rds -> vpnManager.addExtraRoute(
1792                                                         vpnUtil.getVpnName(vpn.getVpnId()),
1793                                                         adj.getIpAddress(), nh, rds,
1794                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1795                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1796                                                         opAdjacency, encapType, writeConfigTxn));
1797                                 }
1798                             });
1799                         }
1800                     }
1801                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1802                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1803                             currVpnIntf.getName(), vpnName);
1804
1805                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1806
1807                     writeOperTxn.merge(
1808                             LogicalDatastoreType.OPERATIONAL,
1809                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1810                                     prefix), VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(),
1811                                     prefix, adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1812
1813                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1814                             adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1815                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1816
1817                     input.setRd(adj.getVrfId());
1818                 }
1819                 if (operationalAdjacency == null) {
1820                     operationalAdjacency = populator.createOperationalAdjacency(input);
1821                 }
1822                 adjacencies.add(operationalAdjacency);
1823                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1824                 VpnInterfaceOpDataEntry newVpnIntf =
1825                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1826                                 aug, dpnId, currVpnIntf.getLportTag(),
1827                                 currVpnIntf.getGatewayMacAddress());
1828
1829                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1830             }
1831         } catch (ReadFailedException e) {
1832             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1833                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1834         }
1835     }
1836
1837     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1838         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
1839         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1840     }
1841
1842     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1843             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1844         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1845         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1846         try {
1847             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1848                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1849             if (optVpnInterface.isPresent()) {
1850                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1851                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1852                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1853                         LogicalDatastoreType.OPERATIONAL, path);
1854                 if (optAdjacencies.isPresent()) {
1855                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1856
1857                     if (!adjacencies.isEmpty()) {
1858                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1859                         Iterator<Adjacency> adjIt = adjacencies.iterator();
1860                         while (adjIt.hasNext()) {
1861                             Adjacency adjElem = adjIt.next();
1862                             if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1863                                 String rd = adjElem.getVrfId();
1864                                 if (adj.getNextHopIpList() != null) {
1865                                     for (String nh : adj.getNextHopIpList()) {
1866                                         deleteExtraRouteFromCurrentAndImportingVpns(
1867                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1868                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
1869                                     }
1870                                 } else if (adj.isPhysNetworkFunc()) {
1871                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1872                                             adj.getIpAddress(), adj.getSubnetId());
1873                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1874                                             writeConfigTxn);
1875                                 }
1876                                 break;
1877                             }
1878
1879                         }
1880                     }
1881                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1882                             dpnId, adj.getVrfId());
1883                 } else {
1884                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1885                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1886                 }
1887             }
1888         } catch (ReadFailedException e) {
1889             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1890                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1891         }
1892     }
1893
1894     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1895         String rd, String intfName, WriteTransaction writeConfigTxn, WriteTransaction writeOperTx) {
1896         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
1897         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1898         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1899             String vpnRd = vpn.getVrfId();
1900             if (vpnRd != null) {
1901                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
1902                         writeOperTx);
1903             }
1904         }
1905     }
1906
1907     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1908         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1909             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1910             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1911     }
1912
1913     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1914         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1915             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1916     }
1917
1918     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1919                                                     WriteTransaction writeConfigTxn, String vpnName) {
1920         if (vpnInterface == null) {
1921             return;
1922         }
1923         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
1924         if (adjs == null) {
1925             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1926                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1927             return;
1928         }
1929         for (Adjacency adj : adjs) {
1930             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1931                 String primaryInterfaceIp = adj.getIpAddress();
1932                 String macAddress = adj.getMacAddress();
1933                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1934
1935                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
1936                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1937
1938                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1939                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1940                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1941                         routerInt, label, writeConfigTxn);
1942                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1943                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1944                         macAddress);
1945                 return;
1946             }
1947         }
1948         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1949                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1950                 primaryRd, vpnName);
1951     }
1952
1953     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1954                                                     WriteTransaction writeConfigTxn, String vpnName) {
1955         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
1956         String rd = vpnUtil.getVpnRd(vpnName);
1957         if (adjs != null) {
1958             List<Adjacency> adjsList = adjs.getAdjacency();
1959             for (Adjacency adj : adjsList) {
1960                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1961                     String primaryInterfaceIp = adj.getIpAddress();
1962                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1963                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1964                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1965                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1966                     return;
1967                 }
1968             }
1969         } else {
1970             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1971                     vpnInterface.getName(), rd);
1972         }
1973     }
1974
1975     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1976         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1977             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1978                   vpnName, intefaceData.vpnInterface.getName());
1979             return;
1980         }
1981         final VpnInterfaceKey key = intefaceData.identifier
1982                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1983         final String interfaceName = key.getName();
1984         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1985                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1986         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1987                   intefaceData.identifier, vpnName);
1988         return;
1989     }
1990
1991     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1992                                               VpnInterface vpnInterface, String vpnName) {
1993         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
1994                .get(vpnName);
1995         if (vpnInterfaces == null) {
1996             vpnInterfaces = new ConcurrentLinkedQueue<>();
1997         }
1998         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
1999         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2000         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2001                 vpnInterface.getName(), vpnName);
2002     }
2003
2004     public boolean isVpnInstanceReady(String vpnInstanceName) {
2005         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2006         if (vpnRd == null) {
2007             return false;
2008         }
2009         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2010
2011         return vpnInstanceOpDataEntry != null;
2012     }
2013
2014     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2015         synchronized (vpnInstanceName.intern()) {
2016             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2017                     unprocessedVpnInterfaces.get(vpnInstanceName);
2018             if (vpnInterfaces != null) {
2019                 while (!vpnInterfaces.isEmpty()) {
2020                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2021                     if (hasVpnInstanceCreatedSuccessfully) {
2022                         processSavedInterface(savedInterface, vpnInstanceName);
2023                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2024                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2025                     } else {
2026                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2027                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2028                     }
2029                 }
2030             } else {
2031                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2032             }
2033         }
2034     }
2035
2036     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2037             VpnInterface vpnInterface) {
2038         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2039             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2040                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2041             if (vpnInterfaces != null) {
2042                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2043                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2044                             + "unprocessed list", vpnInterface.getName(),
2045                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2046                 }
2047             } else {
2048                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2049                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2050             }
2051         }
2052     }
2053
2054     public void vpnInstanceIsReady(String vpnInstanceName) {
2055         processSavedInterfaces(vpnInstanceName, true);
2056     }
2057
2058     public void vpnInstanceFailed(String vpnInstanceName) {
2059         processSavedInterfaces(vpnInstanceName, false);
2060     }
2061
2062     private static class UnprocessedVpnInterfaceData {
2063         InstanceIdentifier<VpnInterface> identifier;
2064         VpnInterface vpnInterface;
2065
2066         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2067             this.identifier = identifier;
2068             this.vpnInterface = vpnInterface;
2069         }
2070
2071         @Override
2072         public int hashCode() {
2073             final int prime = 31;
2074             int result = 1;
2075             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2076             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2077             return result;
2078         }
2079
2080         @Override
2081         public boolean equals(Object obj) {
2082             if (this == obj) {
2083                 return true;
2084             }
2085             if (obj == null) {
2086                 return false;
2087             }
2088             if (getClass() != obj.getClass()) {
2089                 return false;
2090             }
2091             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2092             if (identifier == null) {
2093                 if (other.identifier != null) {
2094                     return false;
2095                 }
2096             } else if (!identifier.equals(other.identifier)) {
2097                 return false;
2098             }
2099             if (vpnInterface == null) {
2100                 if (other.vpnInterface != null) {
2101                     return false;
2102                 }
2103             } else if (!vpnInterface.equals(other.vpnInterface)) {
2104                 return false;
2105             }
2106             return true;
2107         }
2108     }
2109
2110     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2111         String primaryRd = vpnUtil.getVpnRd(vpnName);
2112         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2113         if (vpnInstanceOpData == null) {
2114             return;
2115         }
2116         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2117         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2118             return;
2119         }
2120         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2121         vpnToDpnLists.forEach(vpnToDpnList -> {
2122             if (vpnToDpnList.getVpnInterfaces() == null) {
2123                 return;
2124             }
2125             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2126                 try {
2127                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2128                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2129                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2130                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2131                     if (!vpnInterfaceOptional.isPresent()) {
2132                         return;
2133                     }
2134                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2135                             vpnInterface.getInterfaceName());
2136                     if (configVpnAdjacencies == null) {
2137                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2138                         return;
2139                     }
2140                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2141                             .augmentation(AdjacenciesOp.class).getAdjacency();
2142                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2143                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2144                     // in operational DS. These unprocessed adjacencies will be handled below.
2145                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2146                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2147                     configVpnAdjacencies.stream()
2148                         .filter(adjacency -> operationVpnAdjacencies.stream()
2149                                 .noneMatch(operationalAdjacency ->
2150                                         operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2151                         .forEach(adjacency -> {
2152                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2153                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2154                                 () -> {
2155                                     // TODO Deal with sequencing — the config tx must only submitted
2156                                     // if the oper tx goes in
2157                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2158                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2159                                         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
2160                                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
2161                                                 confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2162                                                     primaryRd, adjacency, vpnInterfaceOptional.get()
2163                                                         .getDpnId(), confTx, operTx)))));
2164                                         return futures;
2165                                     } else {
2166                                         return Collections.emptyList();
2167                                     }
2168                                 });
2169                         });
2170                 } catch (ReadFailedException e) {
2171                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2172                             vpnName, primaryRd);
2173                 }
2174             });
2175         });
2176     }
2177
2178     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2179         private final String interfaceName;
2180         private final boolean add;
2181         private final String txnDestination;
2182
2183         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2184             this.interfaceName = interfaceName;
2185             this.add = add;
2186             this.txnDestination = transactionDest;
2187         }
2188
2189         @Override
2190         public void onSuccess(Void voidObj) {
2191             if (add) {
2192                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2193                         interfaceName, txnDestination);
2194             } else {
2195                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2196             }
2197         }
2198
2199         @Override
2200         public void onFailure(Throwable throwable) {
2201             if (add) {
2202                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2203                         interfaceName, txnDestination, throwable);
2204             } else {
2205                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2206                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2207             }
2208         }
2209     }
2210 }