vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  44 import org.opendaylight.genius.mdsalutil.NWUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.infrautils.caches.CacheProvider;
  49 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  50 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  52 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  58 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  59 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  60 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  61 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 108 import org.slf4j.Logger;
 109 import org.slf4j.LoggerFactory;
 110
 111 @Singleton
 112 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 113
 114     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 115     private static final short DJC_MAX_RETRIES = 3;
 116
 117     private final DataBroker dataBroker;
 118     private final ManagedNewTransactionRunner txRunner;
 119     private final IBgpManager bgpManager;
 120     private final IFibManager fibManager;
 121     private final IMdsalApiManager mdsalManager;
 122     private final IdManagerService idManager;
 123     private final OdlInterfaceRpcService ifaceMgrRpcService;
 124     private final VpnFootprintService vpnFootprintService;
 125     private final IInterfaceManager interfaceManager;
 126     private final IVpnManager vpnManager;
 127     private final ArpResponderHandler arpResponderHandler;
 128     private final JobCoordinator jobCoordinator;
 129
 130     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 131
 132     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 133             new ConcurrentHashMap<>();
 134
 135     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 136
 137     @Inject
 138     public VpnInterfaceManager(final DataBroker dataBroker,
 139                                final IBgpManager bgpManager,
 140                                final IdManagerService idManager,
 141                                final IMdsalApiManager mdsalManager,
 142                                final IFibManager fibManager,
 143                                final OdlInterfaceRpcService ifaceMgrRpcService,
 144                                final VpnFootprintService vpnFootprintService,
 145                                final IInterfaceManager interfaceManager,
 146                                final IVpnManager vpnManager,
 147                                final ArpResponderHandler arpResponderHandler,
 148                                final JobCoordinator jobCoordinator,
 149                                final CacheProvider cacheProvider) {
 150         super(VpnInterface.class, VpnInterfaceManager.class);
 151
 152         this.dataBroker = dataBroker;
 153         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 154         this.bgpManager = bgpManager;
 155         this.idManager = idManager;
 156         this.mdsalManager = mdsalManager;
 157         this.fibManager = fibManager;
 158         this.ifaceMgrRpcService = ifaceMgrRpcService;
 159         this.vpnFootprintService = vpnFootprintService;
 160         this.interfaceManager = interfaceManager;
 161         this.vpnManager = vpnManager;
 162         this.arpResponderHandler = arpResponderHandler;
 163         this.jobCoordinator = jobCoordinator;
 164
 165         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 166                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 167                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 168     }
 169
 170     public Runnable isNotifyTaskQueued(String intfName) {
 171         return vpnIntfMap.remove(intfName);
 172     }
 173
 174     @PostConstruct
 175     public void start() {
 176         LOG.info("{} start", getClass().getSimpleName());
 177         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 178     }
 179
 180     @Override
 181     @PreDestroy
 182     public void close() {
 183         super.close();
 184         vpnInstanceOpDataEntryCache.close();
 185     }
 186
 187     @Override
 188     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 189         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 190     }
 191
 192     @Override
 193     protected VpnInterfaceManager getDataTreeChangeListener() {
 194         return VpnInterfaceManager.this;
 195     }
 196
 197     @Override
 198     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 199         LOG.info("add: intfName {} onto vpnName {}",
 200                  vpnInterface.getName(),
 201                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 202         addVpnInterface(identifier, vpnInterface, null, null);
 203     }
 204
 205     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 206                           final VpnInterface vpnInterface, String vpnName) {
 207         synchronized (vpnName.intern()) {
 208             if (isVpnInstanceReady(vpnName)) {
 209                 return true;
 210             }
 211             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 212             return false;
 213         }
 214     }
 215
 216     // TODO Clean up the exception handling
 217     @SuppressWarnings("checkstyle:IllegalCatch")
 218     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 219                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 220         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 221             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 222             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 223         }
 224     }
 225
 226     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 227                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 228         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 229         final String interfaceName = key.getName();
 230
 231         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 232             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 233                   vpnName, vpnInterface.getName());
 234             return;
 235         }
 236         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 237              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 238         List<Adjacency> copyOldAdjs = null;
 239         if (oldAdjs != null) {
 240             copyOldAdjs = new ArrayList<>();
 241             copyOldAdjs.addAll(oldAdjs);
 242         }
 243         List<Adjacency> copyNewAdjs = null;
 244         if (newAdjs != null) {
 245             copyNewAdjs = new ArrayList<>();
 246             copyNewAdjs.addAll(newAdjs);
 247         }
 248         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 249     }
 250
 251     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 252                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 253                     final List<Adjacency> newAdjs,
 254                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 255         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 256         final String interfaceName = key.getName();
 257         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 258         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 259             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 260             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 261             if (interfaceState != null) {
 262                 try {
 263                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 264                     final int ifIndex = interfaceState.getIfIndex();
 265                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 266                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 267                         // (the inventory tx goes in last)
 268                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 269                         ListenableFuture<Void> confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 270                             confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 271                                 operTx -> futures.add(
 272                                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> {
 273                                             LOG.info(
 274                                                     "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 275                                                     + " on dpn {}",
 276                                                     vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 277                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 278                                                     confTx, operTx, invTx, interfaceState, vpnName);
 279                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 280                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 281                                                         + " Update for swapping VPN {} case.", interfaceName, vpnName);
 282                                                 if (newAdjs != null) {
 283                                                     for (Adjacency adj : newAdjs) {
 284                                                         if (oldAdjs.contains(adj)) {
 285                                                             oldAdjs.remove(adj);
 286                                                         } else {
 287                                                             if (!isBgpVpnInternetVpn
 288                                                                     || VpnUtil.isAdjacencyEligibleToVpnInternet(
 289                                                                     dataBroker, adj)) {
 290                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 291                                                                         primaryRd, adj, dpnId, operTx, confTx);
 292                                                             }
 293                                                         }
 294                                                     }
 295                                                 }
 296                                                 for (Adjacency adj : oldAdjs) {
 297                                                     if (!isBgpVpnInternetVpn
 298                                                             || VpnUtil.isAdjacencyEligibleToVpnInternet(
 299                                                             dataBroker, adj)) {
 300                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 301                                                                 operTx, confTx);
 302                                                     }
 303                                                 }
 304                                             }
 305                                         })))));
 306                         futures.add(confFuture);
 307                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 308                                 MoreExecutors.directExecutor());
 309                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 310                                 + " processed successfully", interfaceName, vpnName, dpnId);
 311                         return futures;
 312                     });
 313                 } catch (NumberFormatException | IllegalStateException e) {
 314                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 315                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 316                                     vpnName, e);
 317                     return;
 318                 }
 319             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 320                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 321                     () -> {
 322                         ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
 323                             createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 324                                     confTx, vpnName);
 325                             LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 326                                     vpnName, vpnInterface.getDpnId());
 327                         });
 328                         ListenableFutures.addErrorLogging(future, LOG,
 329                                 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 330                         return Collections.singletonList(future);
 331                     });
 332             } else {
 333                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 334                         + " is not available", interfaceName, vpnName);
 335             }
 336         } else {
 337             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 338                       + " as vpn is pending delete", interfaceName, vpnName,
 339                     vpnInterface.getDpnId());
 340         }
 341     }
 342
 343     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 344     // see comments below.
 345     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 346     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 347             final int lportTag, boolean isInterfaceUp,
 348             WriteTransaction writeConfigTxn,
 349             WriteTransaction writeOperTxn,
 350             WriteTransaction writeInvTxn,
 351             Interface interfaceState,
 352             final String vpnName) {
 353         final String interfaceName = vpnInterface.getName();
 354         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 355                                                          interfaceName, vpnName);
 356         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 357         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 358         if (!isInterfaceUp) {
 359             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 360                      interfaceName, dpId, vpnName);
 361             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 362             if (vpnId == VpnConstants.INVALID_ID) {
 363                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 364                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 365                         dpId);
 366                 return;
 367             }
 368
 369             boolean waitForVpnInterfaceOpRemoval = false;
 370             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 371                 String opVpnName = opVpnInterface.getVpnInstanceName();
 372                 String primaryInterfaceIp = null;
 373                 if (opVpnName.equals(vpnName)) {
 374                     // Please check if the primary VRF Entry does not exist for VPNInterface
 375                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 376                     // back to back
 377                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 378                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 379                     if (adjs == null) {
 380                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 381                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 382                                 vpnName);
 383                         return;
 384                     }
 385                     for (Adjacency adj : adjs) {
 386                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 387                             primaryInterfaceIp = adj.getIpAddress();
 388                             break;
 389                         }
 390                     }
 391                     if (primaryInterfaceIp == null) {
 392                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 393                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 394                                 dpId, vpnName);
 395                         return;
 396                     }
 397                     // Get the rd of the vpn instance
 398                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 399                     if (vrf != null) {
 400                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 401                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 402                         return;
 403                     }
 404                     waitForVpnInterfaceOpRemoval = true;
 405                 } else {
 406                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 407                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 408                 }
 409             }
 410             if (!waitForVpnInterfaceOpRemoval) {
 411                 // Add the VPNInterface and quit
 412                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 413                         null/*ipAddressSourceValuePair*/,
 414                         true /* add */);
 415                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 416                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 417                 if (!isBgpVpnInternetVpn) {
 418                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 419                             jobCoordinator);
 420                 }
 421                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 422                         dpId, vpnName);
 423                 if (interfaceManager.isExternalInterface(interfaceName)) {
 424                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 425                         NwConstants.ADD_FLOW);
 426                 }
 427                 return;
 428             }
 429
 430             // FIB didn't get a chance yet to clean up this VPNInterface
 431             // Let us give it a chance here !
 432             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 433                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 434             try {
 435                 Runnable notifyTask = new VpnNotifyTask();
 436                 synchronized (notifyTask) {
 437                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 438                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 439                     // published here while holding the lock on it so this path will hit the wait before notify can be
 440                     // invoked.
 441                     vpnIntfMap.put(interfaceName, notifyTask);
 442                     try {
 443                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 444                     } catch (InterruptedException e) {
 445                         // Ignored
 446                     }
 447                 }
 448             } finally {
 449                 vpnIntfMap.remove(interfaceName);
 450             }
 451
 452             if (opVpnInterface != null) {
 453                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 454                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 455                         dpId, vpnName);
 456                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
 457                 return;
 458             }
 459             // VPNInterface got removed, proceed with Add
 460             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 461                     interfaceName, dpId, vpnName);
 462             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 463                     null/*ipAddressSourceValuePair*/,
 464                     true /* add */);
 465             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 466                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 467             if (!isBgpVpnInternetVpn) {
 468                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 469             }
 470             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 471                     + " FIB to clean up", interfaceName, dpId, vpnName);
 472             if (interfaceManager.isExternalInterface(interfaceName)) {
 473                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 474                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 475             }
 476         } else {
 477             try {
 478                 // Interface is retained in the DPN, but its Link Up.
 479                 // Advertise prefixes again for this interface to BGP
 480                 InstanceIdentifier<VpnInterface> identifier =
 481                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 482                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 483                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 484                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 485                 // Perform similar operation as interface add event for extraroutes.
 486                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 487                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 488                         LogicalDatastoreType.CONFIGURATION, path);
 489                 if (!optAdjacencies.isPresent()) {
 490                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 491                     return;
 492                 }
 493                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 494                 for (Adjacency adjacency : adjacencies) {
 495                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 496                         continue;
 497                     }
 498                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 499                     if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 500                             dataBroker, adjacency)) {
 501                         continue;
 502                     }
 503                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 504                             dpId, writeOperTxn, writeConfigTxn);
 505                 }
 506             } catch (ReadFailedException e) {
 507                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 508                         interfaceName, vpnName, primaryRd, dpId);
 509             }
 510         }
 511     }
 512
 513     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 514             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 515         Uuid extNetworkId;
 516         try {
 517             // vpn instance of ext-net interface is the network-id
 518             extNetworkId = new Uuid(vpnName);
 519         } catch (IllegalArgumentException e) {
 520             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 521                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 522             return;
 523         }
 524
 525         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 526         if (routerIds == null || routerIds.isEmpty()) {
 527             LOG.info("processExternalVpnInterface: No router is associated with {}."
 528                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 529                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 530             return;
 531         }
 532
 533         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 534                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 535         for (Uuid routerId : routerIds) {
 536             String routerName = routerId.getValue();
 537             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 538             if (Objects.equals(primarySwitch, dpId)) {
 539                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 540                 if (router != null) {
 541                     if (addOrRemove == NwConstants.ADD_FLOW) {
 542                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 543                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 544                                 dpId, interfaceName, lportTag, writeInvTxn);
 545                     } else {
 546                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 547                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 548                                 dpId, interfaceName, lportTag);
 549                     }
 550                 } else {
 551                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 552                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 553                             interfaceName, dpId, vpnName);
 554                 }
 555             }
 556         }
 557     }
 558
 559     // TODO Clean up the exception handling
 560     @SuppressWarnings("checkstyle:IllegalCatch")
 561     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 562                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 563                                                   String vpnName, String interfaceName) {
 564         if (rd == null) {
 565             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 566                   interfaceName, dpnId, vpnName);
 567             return;
 568         } else {
 569             if (rd.equals(vpnName)) {
 570                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 571                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 572                 return;
 573             }
 574         }
 575         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 576                 interfaceName, dpnId, vpnName, rd);
 577
 578         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 579         if (nextHopIp == null) {
 580             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 581                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 582                     rd, vpnName);
 583             return;
 584         }
 585
 586         try {
 587             //Read NextHops
 588             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 589             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 590                     LogicalDatastoreType.OPERATIONAL, path);
 591             if (adjacencies.isPresent()) {
 592                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 593                 if (!nextHops.isEmpty()) {
 594                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 595                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 596                     VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 597                     long l3vni = vpnInstanceOpData.getL3vni();
 598                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 599                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 600                     for (Adjacency nextHop : nextHops) {
 601                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 602                             continue;
 603                         }
 604                         String gatewayMac = null;
 605                         long label = 0;
 606                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 607                             final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 608                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 609                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 610                                     .get();
 611                         } else {
 612                             label = nextHop.getLabel();
 613                         }
 614                         try {
 615                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 616                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 617                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 618                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 619                                     gatewayMac);
 620                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 621                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 622                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 623                         } catch (Exception e) {
 624                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 625                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 626                                     interfaceName, dpnId, e);
 627                         }
 628                     }
 629                 }
 630             }
 631         } catch (ReadFailedException e) {
 632             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 633                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 634         }
 635     }
 636
 637     // TODO Clean up the exception handling
 638     @SuppressWarnings("checkstyle:IllegalCatch")
 639     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 640                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 641         //Read NextHops
 642         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 643         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 644         if (rd == null) {
 645             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 646                 interfaceName, vpnName);
 647             return;
 648         } else {
 649             if (rd.equals(vpnName)) {
 650                 LOG.info(
 651                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 652                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 653                 return;
 654             }
 655         }
 656         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 657                vpnName, rd);
 658         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 659         try {
 660             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 661                     path);
 662         } catch (ReadFailedException e) {
 663             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 664                     interfaceName, vpnName);
 665         }
 666         if (adjacencies.isPresent()) {
 667             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 668
 669             if (!nextHops.isEmpty()) {
 670                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 671                         nextHops, interfaceName, vpnName, rd);
 672                 for (Adjacency nextHop : nextHops) {
 673                     try {
 674                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 675                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 676                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 677                                     interfaceName, vpnName);
 678                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 679                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 680                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 681                                     interfaceName, vpnName);
 682                         } else {
 683                             // Perform similar operation as interface delete event for extraroutes.
 684                             String allocatedRd = nextHop.getVrfId();
 685                             for (String nh : nextHop.getNextHopIpList()) {
 686                                 deleteExtraRouteFromCurrentAndImportingVpns(
 687                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 688                             }
 689                         }
 690                     } catch (Exception e) {
 691                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 692                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 693                     }
 694                 }
 695             }
 696         }
 697     }
 698
 699     @SuppressWarnings("checkstyle:IllegalCatch")
 700     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 701                                                   String primaryRd, String interfaceName, final long vpnId,
 702                                                   WriteTransaction writeConfigTxn,
 703                                                   WriteTransaction writeOperTxn,
 704                                                   final WriteTransaction writeInvTxn,
 705                                                   Interface interfaceState) {
 706         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 707         // Read NextHops
 708         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 709         Optional<Adjacencies> adjacencies = Optional.absent();
 710         try {
 711             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 712                     path);
 713         } catch (ReadFailedException e) {
 714             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 715                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 716         }
 717         if (!adjacencies.isPresent()) {
 718             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 719                     null/*gwMac*/, writeOperTxn);
 720             return;
 721         }
 722         // Get the rd of the vpn instance
 723         String nextHopIp = null;
 724         try {
 725             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 726         } catch (Exception e) {
 727             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 728                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 729         }
 730         List<String> nhList = new ArrayList<>();
 731         if (nextHopIp != null) {
 732             nhList.add(nextHopIp);
 733             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 734                     interfaceName, dpnId, vpnName, nhList);
 735         }
 736         Optional<String> gwMac = Optional.absent();
 737         String vpnInterfaceSubnetGwMacAddress = null;
 738         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 739         Long l3vni = vpnInstanceOpData.getL3vni();
 740         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 741         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 742         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 743         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 744         List<Adjacency> value = new ArrayList<>();
 745         for (Adjacency nextHop : nextHops) {
 746             String rd = primaryRd;
 747             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 748             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 749                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 750                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 751                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 752                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 753                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 754                 continue;
 755             }
 756             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 757                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 758                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 759                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 760                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 761                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 762                 writeOperTxn.merge(
 763                     LogicalDatastoreType.OPERATIONAL,
 764                     VpnUtil.getPrefixToInterfaceIdentifier(
 765                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 766                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 767                             prefixCue), true);
 768                 final Uuid subnetId = nextHop.getSubnetId();
 769
 770                 String gatewayIp = nextHop.getSubnetGatewayIp();
 771                 if (gatewayIp == null) {
 772                     Optional<String> gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 773                     if (gatewayIpOptional.isPresent()) {
 774                         gatewayIp = gatewayIpOptional.get();
 775                     }
 776                 }
 777
 778                 if (gatewayIp != null) {
 779                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 780                     if (gwMac.isPresent()) {
 781                         // A valid mac-address is available for this subnet-gateway-ip
 782                         // Use this for programming ARP_RESPONDER table here.  And save this
 783                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 784                         // to populate L3_GW_MAC_TABLE there.
 785                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 786                                 gatewayIp, gwMac.get());
 787                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 788                     } else {
 789                         // A valid mac-address is not available for this subnet-gateway-ip
 790                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 791                         // Save this connected-mac-address as gateway-mac-address for the
 792                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 793                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 794                         if (gwMac.isPresent()) {
 795                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 796                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 797                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 798                                     gatewayIp, gwMac.get());
 799                         } else {
 800                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 801                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 802                                 + "gwIp {}",
 803                                 subnetId, interfaceName, vpnName, gatewayIp);
 804                         }
 805                     }
 806                 } else {
 807                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 808                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 809                         subnetId, interfaceName, vpnName);
 810                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 811                 }
 812                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 813                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 814             } else {
 815                 //Extra route adjacency
 816                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 817                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 818                 synchronized (vpnPrefixKey.intern()) {
 819                     java.util.Optional<String> rdToAllocate = VpnUtil
 820                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 821                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 822                     if (rdToAllocate.isPresent()) {
 823                         rd = rdToAllocate.get();
 824                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 825                             rd, prefix);
 826                     } else {
 827                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 828                         continue;
 829                     }
 830                 }
 831                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 832                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 833                         interfaceName, dpnId);
 834             }
 835             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 836             // can be different from the gateway-mac-address within the VRFEntry as the
 837             // gateway-mac-address is a superset.
 838             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 839                     : RouteOrigin.STATIC;
 840             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 841                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 842                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 843             Adjacency operationalAdjacency = null;
 844             try {
 845                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 846             } catch (NullPointerException e) {
 847                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 848                     input, e.getMessage());
 849                 return;
 850             }
 851             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 852                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 853                         vpnName, l3vni, origin,
 854                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 855             }
 856             value.add(operationalAdjacency);
 857         }
 858
 859         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 860         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 861                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 862
 863         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 864                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 865                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 866
 867         for (Adjacency nextHop : aug.getAdjacency()) {
 868             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 869             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 870                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 871                         : RouteOrigin.STATIC;
 872                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 873                 registeredPopulator.populateFib(input, writeConfigTxn);
 874             }
 875         }
 876     }
 877
 878     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 879                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 880         VpnInterfaceOpDataEntry opInterface =
 881               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 882         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 883             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 884         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 885                 WriteTransaction.CREATE_MISSING_PARENTS);
 886         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 887                 interfaceName, dpnId, vpnName);
 888     }
 889
 890     // TODO Clean up the exception handling
 891     @SuppressWarnings("checkstyle:IllegalCatch")
 892     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 893                                            StateTunnelList stateTunnelList,
 894                                            WriteTransaction writeConfigTxn,
 895                                            WriteTransaction writeOperTxn) {
 896
 897         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 898         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 899         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 900         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 901         if (adjList.isEmpty()) {
 902             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 903                     vpnInterface, srcDpnId);
 904             return;
 905         }
 906         String prefix = null;
 907         long label = 0;
 908         List<Adjacency> value = new ArrayList<>();
 909         boolean isNextHopAddReqd = false;
 910         String vpnName = vpnInterface.getVpnInstanceName();
 911         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 912         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 913         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 914                 vpnInterface.getName(), vpnInterface.getDpnId(),
 915                 vpnInterface.getVpnInstanceName(), adjList);
 916         for (Adjacency adj : adjList) {
 917             String rd = adj.getVrfId();
 918             rd = rd != null ? rd : vpnName;
 919             prefix = adj.getIpAddress();
 920             label = adj.getLabel();
 921             List<String> nhList = Collections.singletonList(srcTepIp);
 922             List<String> nextHopList = adj.getNextHopIpList();
 923             // If TEP is added , update the nexthop of primary adjacency.
 924             // Secondary adj nexthop is already pointing to primary adj IP address.
 925             if (nextHopList != null && !nextHopList.isEmpty()) {
 926                 /* everything right already */
 927             } else {
 928                 isNextHopAddReqd = true;
 929             }
 930
 931             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 932                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 933             } else {
 934                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 935                 if (!vrfEntryOptional.isPresent()) {
 936                     continue;
 937                 }
 938                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 939                 if (!nhList.contains(srcTepIp)) {
 940                     nhList.add(srcTepIp);
 941                     isNextHopAddReqd = true;
 942                 }
 943                 value.add(adj);
 944             }
 945
 946             if (isNextHopAddReqd) {
 947                 updateLabelMapper(label, nhList);
 948                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 949                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 950                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 951                 // Update the VRF entry with nextHop
 952                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 953                         label, true, writeConfigTxn);
 954
 955                 //Get the list of VPN's importing this route(prefix) .
 956                 // Then update the VRF entry with nhList
 957                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 958                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 959                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 960                     String vpnRd = vpn.getVrfId();
 961                     if (vpnRd != null) {
 962                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 963                                 srcTepIp, label, true, writeConfigTxn);
 964                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 965                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 966                             vpnInterface.getName(), srcDpnId, vpnName,
 967                             vpn.getVpnInstanceName(), vpnRd);
 968                     }
 969                 }
 970                 // Advertise the prefix to BGP only for external vpn
 971                 // since there is a nexthop change.
 972                 try {
 973                     if (!rd.equalsIgnoreCase(vpnName)) {
 974                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 975                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 976                                 null /*gatewayMacAddress*/);
 977                     }
 978                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 979                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 980                             srcDpnId, vpnName);
 981                 } catch (Exception ex) {
 982                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 983                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 984                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 985                 }
 986             }
 987         }
 988         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 989         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 990                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 991                 .addAugmentation(AdjacenciesOp.class, aug).build();
 992         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 993                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 994         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 995                 WriteTransaction.CREATE_MISSING_PARENTS);
 996         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 997                 vpnInterface.getName(), srcDpnId, vpnName);
 998
 999     }
1000
1001     // TODO Clean up the exception handling
1002     @SuppressWarnings("checkstyle:IllegalCatch")
1003     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1004                                               StateTunnelList stateTunnelList,
1005                                               WriteTransaction writeConfigTxn,
1006                                               WriteTransaction writeOperTxn) {
1007
1008         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1009         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1010         String prefix = null;
1011         long label = 0;
1012         boolean isNextHopRemoveReqd = false;
1013         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1014         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1015         String vpnName = vpnInterface.getVpnInstanceName();
1016         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1017         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1018         if (adjList != null) {
1019             List<Adjacency> value = new ArrayList<>();
1020             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1021                     vpnInterface.getName(), vpnInterface.getDpnId(),
1022                     vpnInterface.getVpnInstanceName(), adjList);
1023             for (Adjacency adj : adjList) {
1024                 List<String> nhList = new ArrayList<>();
1025                 String rd = adj.getVrfId();
1026                 rd = rd != null ? rd : vpnName;
1027                 prefix = adj.getIpAddress();
1028                 List<String> nextHopList = adj.getNextHopIpList();
1029                 label = adj.getLabel();
1030                 if (nextHopList != null && !nextHopList.isEmpty()) {
1031                     isNextHopRemoveReqd = true;
1032                 }
1033                 // If TEP is deleted , remove the nexthop from primary adjacency.
1034                 // Secondary adj nexthop will continue to point to primary adj IP address.
1035                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1036                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1037                 } else {
1038                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1039                     if (!vrfEntryOptional.isPresent()) {
1040                         continue;
1041                     }
1042                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1043                     if (nhList.contains(srcTepIp)) {
1044                         nhList.remove(srcTepIp);
1045                         isNextHopRemoveReqd = true;
1046                     }
1047                     value.add(adj);
1048                 }
1049
1050                 if (isNextHopRemoveReqd) {
1051                     updateLabelMapper(label, nhList);
1052                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1053                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1054                             prefix, nhList, vpnName,
1055                             vpnId, rd, vpnInterface.getName());
1056                     // Update the VRF entry with removed nextHop
1057                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1058                             label, false, writeConfigTxn);
1059
1060                     //Get the list of VPN's importing this route(prefix) .
1061                     // Then update the VRF entry with nhList
1062                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1063                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1064                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1065                         String vpnRd = vpn.getVrfId();
1066                         if (vpnRd != null) {
1067                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1068                                     srcTepIp, label, false, writeConfigTxn);
1069                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1070                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1071                                     nhList, label, vpnInterface.getName(), srcDpnId,
1072                                     vpnName,
1073                                     vpn.getVpnInstanceName(), vpnRd);
1074                         }
1075                     }
1076
1077                     // Withdraw prefix from BGP only for external vpn.
1078                     try {
1079                         if (!rd.equalsIgnoreCase(vpnName)) {
1080                             bgpManager.withdrawPrefix(rd, prefix);
1081                         }
1082                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1083                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1084                                 vpnInterface.getName(), srcDpnId,
1085                                 vpnName);
1086                     } catch (Exception ex) {
1087                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1088                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1089                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1090                     }
1091                 }
1092             }
1093             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1094             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1095                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1096                     .addAugmentation(AdjacenciesOp.class, aug).build();
1097             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1098                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1099             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1100                     WriteTransaction.CREATE_MISSING_PARENTS);
1101             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1102                          vpnInterface.getName(), srcDpnId, vpnName);
1103         }
1104     }
1105
1106     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1107         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1108
1109         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1110         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1111         if (vpnInstanceOpDataEntry == null) {
1112             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1113                     + " to check for vpns exporting the routes", vpnName);
1114             return vpnsToExportRoute;
1115         }
1116
1117         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1118             if (input.getVpnInstanceName() == null) {
1119                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1120                         input.getVrfId());
1121                 return false;
1122             }
1123             return !input.getVpnInstanceName().equals(vpnName);
1124         };
1125
1126         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1127             Iterable<String> commonRTs =
1128                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1129                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1130             return Iterators.size(commonRTs.iterator()) > 0;
1131         };
1132
1133         vpnsToExportRoute =
1134                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1135                         Collectors.toList());
1136         return vpnsToExportRoute;
1137     }
1138
1139     // TODO Clean up the exception handling
1140     @SuppressWarnings("checkstyle:IllegalCatch")
1141     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1142         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1143         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1144             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1145             if (vrfEntries != null) {
1146                 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1147                     for (VrfEntry vrfEntry : vrfEntries) {
1148                         try {
1149                             if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1150                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1151                                 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1152                                                 + " is not a controller managed non intervpn link route. Ignoring.",
1153                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1154                                 continue;
1155                             }
1156                             String prefix = vrfEntry.getDestPrefix();
1157                             String gwMac = vrfEntry.getGatewayMacAddress();
1158                             vrfEntry.getRoutePaths().forEach(routePath -> {
1159                                 String nh = routePath.getNexthopAddress();
1160                                 int label = routePath.getLabel().intValue();
1161                                 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1162                                         vrfEntry.getOrigin()))) {
1163                                     LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1164                                                     + " nexthop {} label {} to vpn {} vpnRd {}",
1165                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1166                                     fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1167                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1168                                             0 /*l3vni*/, gwMac,  vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1169                                             confTx);
1170                                 } else {
1171                                     LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1172                                                     + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1173                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1174                                     SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1175                                     importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1176                                             confTx);
1177                                 }
1178                             });
1179                         } catch (RuntimeException e) {
1180                             LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1181                                             + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1182                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1183                         }
1184                     }
1185                 }), LOG, "Error handing VPN exporting routes");
1186             } else {
1187                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1188                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1189             }
1190         }
1191     }
1192
1193     @Override
1194     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1195         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1196         final String interfaceName = key.getName();
1197         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1198             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1199             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1200         }
1201     }
1202
1203     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1204                                 final VpnInterface vpnInterface, final String vpnName,
1205                                 final String interfaceName) {
1206         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1207             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1208                 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1209                     deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1210                     LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1211                 });
1212                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1213                         vpnInterface.getName(), vpnName);
1214                 return Collections.singletonList(future);
1215             }, DJC_MAX_RETRIES);
1216         } else {
1217             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1218             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1219         }
1220     }
1221
1222     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1223     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1224                                 final VpnInterface vpnInterface, final String vpnName,
1225                                 final String interfaceName, final Interface interfaceState) {
1226         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1227                 vpnName, vpnInterface.getDpnId());
1228         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1229         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1230             () -> {
1231                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1232                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1233                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1234                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1235                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1236                                     interfaceName, vpnName);
1237                             BigInteger dpId = BigInteger.ZERO;
1238                             int ifIndex = 0;
1239                             String gwMacAddress = null;
1240                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1241                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1242                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1243                             try {
1244                                 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1245                                         LogicalDatastoreType.OPERATIONAL, interfaceId);
1246                             } catch (ReadFailedException e) {
1247                                 LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName,
1248                                         vpnName);
1249                                 return;
1250                             }
1251                             if (interfaceState != null) {
1252                                 try {
1253                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1254                                 } catch (NumberFormatException | IllegalStateException e) {
1255                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1256                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1257                                                     + " from vpn interface op data store. ", interfaceName,
1258                                             vpnInterface.getDpnId(), vpnName, e);
1259                                     dpId = BigInteger.ZERO;
1260                                 }
1261                                 ifIndex = interfaceState.getIfIndex();
1262                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1263                             } else {
1264                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1265                                         + " from vpn interface op.", interfaceName);
1266                                 if (optVpnInterface.isPresent()) {
1267                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1268                                     dpId = vpnOpInterface.getDpnId();
1269                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1270                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1271                                 } else {
1272                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1273                                                     + " as interfaceState and vpn interface op is not"
1274                                             + " available", interfaceName, vpnName);
1275                                     return;
1276                                 }
1277                             }
1278                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1279                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1280                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1281                             LOG.info(
1282                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1283                                             + "successfully",
1284                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1285                         })))));
1286                 futures.add(configFuture);
1287                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1288                 return futures;
1289             }, DJC_MAX_RETRIES);
1290     }
1291
1292     protected void processVpnInterfaceDown(BigInteger dpId,
1293                                            String interfaceName,
1294                                            int lportTag,
1295                                            String gwMac,
1296                                            VpnInterfaceOpDataEntry vpnOpInterface,
1297                                            boolean isInterfaceStateDown,
1298                                            WriteTransaction writeConfigTxn,
1299                                            WriteTransaction writeOperTxn,
1300                                            WriteTransaction writeInvTxn) {
1301         if (vpnOpInterface == null) {
1302             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1303                     + " as it is not available in operational data store", interfaceName, dpId);
1304             return;
1305         }
1306         final String vpnName = vpnOpInterface.getVpnInstanceName();
1307         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1308                                                     interfaceName, vpnName);
1309         if (!isInterfaceStateDown) {
1310             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1311             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1312                     null);
1313             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1314             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1315                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1316             if (interfaceManager.isExternalInterface(interfaceName)) {
1317                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1318                         NwConstants.DEL_FLOW);
1319             }
1320             if (!isBgpVpnInternetVpn) {
1321                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1322             }
1323             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1324                     + " successful", interfaceName, dpId, vpnName);
1325         } else {
1326             // Interface is retained in the DPN, but its Link Down.
1327             // Only withdraw the prefixes for this interface from BGP
1328             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1329         }
1330     }
1331
1332     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1333                                           final String vpnName, final long vpnId, String gwMac,
1334                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1335                                           final WriteTransaction writeInvTxn) {
1336         //Read NextHops
1337         try {
1338             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1339                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1340             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1341             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1342                     LogicalDatastoreType.OPERATIONAL, path);
1343             String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1344             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1345                     interfaceName, dpnId, vpnName, primaryRd);
1346             if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1347                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1348                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1349                         interfaceName, dpnId, vpnName, nextHops);
1350                 for (Adjacency nextHop : nextHops) {
1351                     if (nextHop.isPhysNetworkFunc()) {
1352                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1353                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1354                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1355                                 null/*writeCfgTxn*/);
1356                     } else {
1357                         String rd = nextHop.getVrfId();
1358                         List<String> nhList;
1359                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1360                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1361                         } else {
1362                             // This is a primary adjacency
1363                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1364                                     : Collections.emptyList();
1365                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1366                                     interfaceName, writeInvTxn);
1367                         }
1368                         if (!nhList.isEmpty()) {
1369                             if (rd.equals(vpnName)) {
1370                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1371                                 //remove from FIB directly
1372                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1373                                         interfaceName, dpnId, writeConfigTxn));
1374                             } else {
1375                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1376                                         interfaceName, writeConfigTxn);
1377                             }
1378                         } else {
1379                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1380                                             + " interface {}", nextHop.getIpAddress(), rd,
1381                                     nextHop.getAdjacencyType().toString(), interfaceName);
1382                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1383                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1384                         }
1385                     }
1386                     String ip = nextHop.getIpAddress().split("/")[0];
1387                     LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1388                     if (vpnVipToPort != null) {
1389                         VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip, null);
1390                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1391                                         + " ip {} on dpn {} for vpn {} from VpnPortData Entry",
1392                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1393                     }
1394                 }
1395             } else {
1396                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1397                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1398                         + " Removing it.", interfaceName, vpnName, dpnId);
1399                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1400             }
1401         } catch (ReadFailedException e) {
1402             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1403                     interfaceName, dpnId, vpnName);
1404         }
1405     }
1406
1407     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1408                                                                    String interfaceName, BigInteger dpnId,
1409                                                                    WriteTransaction writeConfigTxn) {
1410         return (nh) -> {
1411             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1412                     writeConfigTxn);
1413             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1414                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1415                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1416         };
1417     }
1418
1419     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1420                                            BigInteger dpnId, String rd, String interfaceName,
1421                                            WriteTransaction writeConfigTxn) {
1422         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1423                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1424         nhList.forEach((nh) -> {
1425             //IRT: remove routes from other vpns importing it
1426             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1427                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1428             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1429                 String vpnRd = vpn.getVrfId();
1430                 if (vpnRd != null) {
1431                     fibManager.removeOrUpdateFibEntry(vpnRd,
1432                             nextHop.getIpAddress(), nh, writeConfigTxn);
1433                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1434                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1435                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1436                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1437                 }
1438             }
1439         });
1440     }
1441
1442     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1443                                                  int lportTag, String gwMac, String interfaceName,
1444                                                  WriteTransaction writeInvTxn) {
1445         final Uuid subnetId = nextHop.getSubnetId();
1446         if (nextHop.getSubnetGatewayMacAddress() == null) {
1447             // A valid mac-address was not available for this subnet-gateway-ip
1448             // So a connected-mac-address was used for this subnet and we need
1449             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1450             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1451                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1452         }
1453         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1454                 subnetId);
1455     }
1456
1457     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1458                                                   String interfaceName) {
1459         // This is either an extra-route (or) a learned IP via subnet-route
1460         List<String> nhList = null;
1461         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1462         if (nextHopIp == null || nextHopIp.isEmpty()) {
1463             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1464                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1465                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1466                     vpnName);
1467             nhList = Collections.emptyList();
1468         } else {
1469             nhList = Collections.singletonList(nextHopIp);
1470         }
1471         return nhList;
1472     }
1473
1474     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1475         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1476         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1477         // else use connected interface
1478         if (gwPort != null && gwPort.isSubnetIp()) {
1479             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1480                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1481             return Optional.of(gwPort.getMacAddress());
1482         }
1483         return Optional.absent();
1484     }
1485
1486     @Override
1487     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1488         final VpnInterface update) {
1489         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1490                 update.getDpnId(), original.getVpnInstanceNames(),
1491                 update.getVpnInstanceNames());
1492         final String vpnInterfaceName = update.getName();
1493         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1494         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1495         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1496             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1497         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1498         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1499             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1500
1501         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1502         //handles switching between <internal VPN - external VPN>
1503         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1504             LOG.info("update: handled VPNInterface {} on dpn {} update"
1505                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1506                      original.getName(), dpnId,
1507                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1508                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1509             return;
1510         }
1511         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1512             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1513             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1514             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1515             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1516             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1517                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1518                     // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1519                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1520                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1521                         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(operTx -> {
1522                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1523                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1524                             LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1525                                     update.getName(), newVpnName);
1526                             //handle both addition and removal of adjacencies
1527                             //currently, new adjacency may be an extra route
1528                             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1529                             if (!oldAdjs.equals(newAdjs)) {
1530                                 for (Adjacency adj : copyNewAdjs) {
1531                                     if (copyOldAdjs.contains(adj)) {
1532                                         copyOldAdjs.remove(adj);
1533                                     } else {
1534                                         // add new adjacency - right now only extra route will hit this path
1535                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker,
1536                                                 adj)) {
1537                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1538                                                     dpnId, operTx, confTx);
1539                                         }
1540                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1541                                                 + " vpn interface {} on vpn {} dpnId {}",
1542                                                 adj.getIpAddress(), adj.getNextHopIpList(),
1543                                                 adj.getLabel(), adj.getSubnetId(), update.getName(),
1544                                                 newVpnName, dpnId);
1545                                     }
1546                                 }
1547                                 for (Adjacency adj : copyOldAdjs) {
1548                                     if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker,
1549                                             adj)) {
1550                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1551                                                 && !adj.isPhysNetworkFunc()) {
1552                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1553                                                     operTx, confTx);
1554                                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = operTx.read(
1555                                                     LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier).get();
1556                                             VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get();
1557                                             long vpnId = VpnUtil.getVpnId(dataBroker, newVpnName);
1558                                             VpnUtil.removePrefixToInterfaceAdj(dataBroker, adj, vpnId,
1559                                                     vpnInterfaceOpDataEntry, operTx);
1560                                             //remove FIB entry
1561                                             String vpnRd = VpnUtil.getVpnRd(dataBroker, newVpnName);
1562                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1563                                                     + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1564                                             //remove BGP entry
1565                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1566                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1567                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1568                                             }
1569                                         } else {
1570                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1571                                                     operTx, confTx);
1572                                         }
1573                                     }
1574                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1575                                                     + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1576                                                     .getNextHopIpList(),
1577                                             adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1578                                 }
1579                             }
1580                         }));
1581                     }));
1582                     for (ListenableFuture<Void> future : futures) {
1583                         ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1584                                 update.getName(), update.getVpnInstanceNames());
1585                     }
1586                     return futures;
1587                 });
1588             } else {
1589                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1590                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1591             }
1592         }
1593     }
1594
1595     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1596                                                  VpnInterface original, VpnInterface update) {
1597         boolean isSwap = Boolean.FALSE;
1598         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1599         final String interfaceName = key.getName();
1600         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1601             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1602         List<String> oldVpnListCopy = new ArrayList<>();
1603         oldVpnListCopy.addAll(oldVpnList);
1604         List<String> newVpnList = update.getVpnInstanceNames().stream()
1605             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1606         oldVpnList.removeAll(newVpnList);
1607         newVpnList.removeAll(oldVpnListCopy);
1608         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1609             for (String oldVpnName: oldVpnList) {
1610                 isSwap = Boolean.TRUE;
1611                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1612                         + " running config-driven swap removal", interfaceName, oldVpnName);
1613                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1614                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1615                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1616             }
1617             //Wait for previous interface bindings to be removed
1618             try {
1619                 Thread.sleep(2000);
1620             } catch (InterruptedException e) {
1621                 //Ignore
1622             }
1623             for (String newVpnName: newVpnList) {
1624                 String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1625                 isSwap = Boolean.TRUE;
1626                 if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1627                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1628                             + "running config-driven swap addition", interfaceName, newVpnName);
1629                     final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1630                     final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1631                             ? origAdjs.getAdjacency() : new ArrayList<>();
1632                     final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1633                     final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1634                             ? updateAdjs.getAdjacency() : new ArrayList<>();
1635
1636                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1637                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1638                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1639                             interfaceName, oldVpnListCopy, newVpnName);
1640                 }
1641             }
1642         }
1643         return isSwap;
1644     }
1645
1646     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1647         try {
1648             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1649             synchronized (label.toString().intern()) {
1650                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1651                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1652                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1653                         LogicalDatastoreType.OPERATIONAL, lriIid);
1654                 if (opResult.isPresent()) {
1655                     LabelRouteInfo labelRouteInfo =
1656                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1657                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1658                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1659                 }
1660             }
1661             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1662                     nextHopIpList);
1663         } catch (ReadFailedException e) {
1664             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1665                     nextHopIpList);
1666         } catch (TransactionCommitFailedException e) {
1667             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1668                     nextHopIpList);
1669         }
1670     }
1671
1672     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1673         SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1674
1675         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1676         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1677                 .addAugmentation(SubnetRoute.class, route).build();
1678         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1679         InstanceIdentifierBuilder<VrfTables> idBuilder =
1680             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1681         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1682         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1683         if (writeConfigTxn != null) {
1684             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1685         } else {
1686             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1687         }
1688         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1689                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1690     }
1691
1692     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1693                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1694                                            WriteTransaction writeConfigTxn) {
1695         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1696         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1697         try {
1698             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1699                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1700             if (optVpnInterface.isPresent()) {
1701                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1702                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1703                 String vpnName = currVpnIntf.getVpnInstanceName();
1704                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1705                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1706                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1707                         LogicalDatastoreType.OPERATIONAL, adjPath);
1708                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1709                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1710                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1711                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1712                 List<Adjacency> adjacencies;
1713                 if (optAdjacencies.isPresent()) {
1714                     adjacencies = optAdjacencies.get().getAdjacency();
1715                 } else {
1716                     // This code will be hit in case of first PNF adjacency
1717                     adjacencies = new ArrayList<>();
1718                 }
1719                 long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1720                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1721                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1722                 Adjacency operationalAdjacency = null;
1723                 //Handling dual stack neutron port primary adjacency
1724                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1725                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1726                             currVpnIntf.getName(), vpnName);
1727                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1728                             currVpnIntf.getName());
1729                     if (interfaceState != null) {
1730                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1731                                 currVpnIntf.getName(),
1732                                 vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1733                     }
1734                 }
1735                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1736                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1737                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1738                             : RouteOrigin.STATIC;
1739                     String nh = adj.getNextHopIpList().get(0);
1740                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1741                     synchronized (vpnPrefixKey.intern()) {
1742                         java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1743                                 dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1744                         if (rdToAllocate.isPresent()) {
1745                             input.setRd(rdToAllocate.get());
1746                             operationalAdjacency = populator.createOperationalAdjacency(input);
1747                             int label = operationalAdjacency.getLabel().intValue();
1748                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1749                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1750                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1751                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1752                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1753                                     vpnName, label, currVpnIntf.getName(), dpnId);
1754                         } else {
1755                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1756                                     vpnName, prefix);
1757                             return;
1758                         }
1759                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1760                         // Keeping the MPLS check for now.
1761                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1762                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1763                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1764                                     VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1765                             vpnsToImportRoute.forEach(vpn -> {
1766                                 if (vpn.getVrfId() != null) {
1767                                     VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1768                                             dataBroker, vpn.getVpnId(), vpnId, prefix,
1769                                             VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1770                                             .ifPresent(
1771                                                 rds -> vpnManager.addExtraRoute(
1772                                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1773                                                         adj.getIpAddress(), nh, rds,
1774                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1775                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1776                                                         opAdjacency, encapType, writeConfigTxn));
1777                                 }
1778                             });
1779                         }
1780                     }
1781                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1782                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1783                             currVpnIntf.getName(), vpnName);
1784
1785                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1786
1787                     writeOperTxn.merge(
1788                             LogicalDatastoreType.OPERATIONAL,
1789                             VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1790                                     adj.getSubnetId().getValue()), prefix),
1791                             VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1792                                     adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1793
1794                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1795                             adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1796                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1797
1798                     input.setRd(adj.getVrfId());
1799                 }
1800                 if (operationalAdjacency == null) {
1801                     operationalAdjacency = populator.createOperationalAdjacency(input);
1802                 }
1803                 adjacencies.add(operationalAdjacency);
1804                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1805                 VpnInterfaceOpDataEntry newVpnIntf =
1806                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1807                                 aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1808                                 currVpnIntf.getGatewayMacAddress());
1809
1810                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1811             }
1812         } catch (ReadFailedException e) {
1813             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1814                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1815         }
1816     }
1817
1818     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1819         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1820         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1821     }
1822
1823     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1824             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1825         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1826         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1827         try {
1828             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1829                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1830             if (optVpnInterface.isPresent()) {
1831                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1832                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1833                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1834                         LogicalDatastoreType.OPERATIONAL, path);
1835                 if (optAdjacencies.isPresent()) {
1836                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1837
1838                     if (!adjacencies.isEmpty()) {
1839                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1840                         Iterator<Adjacency> adjIt = adjacencies.iterator();
1841                         while (adjIt.hasNext()) {
1842                             Adjacency adjElem = adjIt.next();
1843                             if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1844                                 String rd = adjElem.getVrfId();
1845                                 adjIt.remove();
1846                                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1847                                 VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1848                                         .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1849                                                 currVpnIntf.getVpnInstanceName(), aug, dpnId,
1850                                                 currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1851                                                 currVpnIntf.getGatewayMacAddress());
1852                                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1853                                 if (adj.getNextHopIpList() != null) {
1854                                     for (String nh : adj.getNextHopIpList()) {
1855                                         deleteExtraRouteFromCurrentAndImportingVpns(
1856                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1857                                                 currVpnIntf.getName(), writeConfigTxn);
1858                                     }
1859                                 } else if (adj.isPhysNetworkFunc()) {
1860                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1861                                             adj.getIpAddress(), adj.getSubnetId());
1862                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1863                                             writeConfigTxn);
1864                                 }
1865                                 break;
1866                             }
1867
1868                         }
1869                     }
1870                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1871                             dpnId, adj.getVrfId());
1872                 } else {
1873                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1874                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1875                 }
1876             }
1877         } catch (ReadFailedException e) {
1878             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1879                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1880         }
1881     }
1882
1883     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1884         String rd, String intfName, WriteTransaction writeConfigTxn) {
1885         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1886         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1887         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1888             String vpnRd = vpn.getVrfId();
1889             if (vpnRd != null) {
1890                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1891             }
1892         }
1893     }
1894
1895     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1896         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1897             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1898             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1899     }
1900
1901     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1902         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1903             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1904     }
1905
1906     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1907                                                     WriteTransaction writeConfigTxn, String vpnName) {
1908         if (vpnInterface == null) {
1909             return;
1910         }
1911         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1912         if (adjs == null) {
1913             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1914                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1915             return;
1916         }
1917         for (Adjacency adj : adjs) {
1918             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1919                 String primaryInterfaceIp = adj.getIpAddress();
1920                 String macAddress = adj.getMacAddress();
1921                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1922
1923                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1924                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1925
1926                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1927                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1928                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1929                         routerInt, label, writeConfigTxn);
1930                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1931                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1932                         macAddress);
1933                 return;
1934             }
1935         }
1936         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1937                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1938                 primaryRd, vpnName);
1939     }
1940
1941     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1942                                                     WriteTransaction writeConfigTxn, String vpnName) {
1943         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
1944         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1945         if (adjs != null) {
1946             List<Adjacency> adjsList = adjs.getAdjacency();
1947             for (Adjacency adj : adjsList) {
1948                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1949                     String primaryInterfaceIp = adj.getIpAddress();
1950                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1951                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1952                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1953                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1954                     return;
1955                 }
1956             }
1957         } else {
1958             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1959                     vpnInterface.getName(), rd);
1960         }
1961     }
1962
1963     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1964         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1965             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1966                   vpnName, intefaceData.vpnInterface.getName());
1967             return;
1968         }
1969         final VpnInterfaceKey key = intefaceData.identifier
1970                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1971         final String interfaceName = key.getName();
1972         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1973                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1974         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1975                   intefaceData.identifier, vpnName);
1976         return;
1977     }
1978
1979     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1980                                               VpnInterface vpnInterface, String vpnName) {
1981         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
1982                .get(vpnName);
1983         if (vpnInterfaces == null) {
1984             vpnInterfaces = new ConcurrentLinkedQueue<>();
1985         }
1986         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
1987         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
1988         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
1989                 vpnInterface.getName(), vpnName);
1990     }
1991
1992     public boolean isVpnInstanceReady(String vpnInstanceName) {
1993         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
1994         if (vpnRd == null) {
1995             return false;
1996         }
1997         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1998
1999         return vpnInstanceOpDataEntry != null;
2000     }
2001
2002     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2003         synchronized (vpnInstanceName.intern()) {
2004             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2005                     unprocessedVpnInterfaces.get(vpnInstanceName);
2006             if (vpnInterfaces != null) {
2007                 while (!vpnInterfaces.isEmpty()) {
2008                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2009                     if (hasVpnInstanceCreatedSuccessfully) {
2010                         processSavedInterface(savedInterface, vpnInstanceName);
2011                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2012                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2013                     } else {
2014                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2015                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2016                     }
2017                 }
2018             } else {
2019                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2020             }
2021         }
2022     }
2023
2024     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2025             VpnInterface vpnInterface) {
2026         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2027             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2028                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2029             if (vpnInterfaces != null) {
2030                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2031                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2032                             + "unprocessed list", vpnInterface.getName(),
2033                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2034                 }
2035             } else {
2036                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2037                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2038             }
2039         }
2040     }
2041
2042     public void vpnInstanceIsReady(String vpnInstanceName) {
2043         processSavedInterfaces(vpnInstanceName, true);
2044     }
2045
2046     public void vpnInstanceFailed(String vpnInstanceName) {
2047         processSavedInterfaces(vpnInstanceName, false);
2048     }
2049
2050     private static class UnprocessedVpnInterfaceData {
2051         InstanceIdentifier<VpnInterface> identifier;
2052         VpnInterface vpnInterface;
2053
2054         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2055             this.identifier = identifier;
2056             this.vpnInterface = vpnInterface;
2057         }
2058
2059         @Override
2060         public int hashCode() {
2061             final int prime = 31;
2062             int result = 1;
2063             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2064             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2065             return result;
2066         }
2067
2068         @Override
2069         public boolean equals(Object obj) {
2070             if (this == obj) {
2071                 return true;
2072             }
2073             if (obj == null) {
2074                 return false;
2075             }
2076             if (getClass() != obj.getClass()) {
2077                 return false;
2078             }
2079             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2080             if (identifier == null) {
2081                 if (other.identifier != null) {
2082                     return false;
2083                 }
2084             } else if (!identifier.equals(other.identifier)) {
2085                 return false;
2086             }
2087             if (vpnInterface == null) {
2088                 if (other.vpnInterface != null) {
2089                     return false;
2090                 }
2091             } else if (!vpnInterface.equals(other.vpnInterface)) {
2092                 return false;
2093             }
2094             return true;
2095         }
2096     }
2097
2098     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2099         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2100         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2101         if (vpnInstanceOpData == null) {
2102             return;
2103         }
2104         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2105         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2106             return;
2107         }
2108         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2109         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2110             try {
2111                 InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2112                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2113                 Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker.syncReadOptional(
2114                         dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2115                 if (!vpnInterfaceOptional.isPresent()) {
2116                     return;
2117                 }
2118                 List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2119                         vpnInterface.getInterfaceName());
2120                 if (configVpnAdjacencies == null) {
2121                     LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2122                     return;
2123                 }
2124                 List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2125                         .augmentation(AdjacenciesOp.class).getAdjacency();
2126                 // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2127                 // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2128                 // in operational DS. These unprocessed adjacencies will be handled below.
2129                 // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2130                 // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2131                 configVpnAdjacencies.stream()
2132                         .filter(adjacency -> operationVpnAdjacencies.stream()
2133                                 .noneMatch(operationalAdjacency ->
2134                                         operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2135                         .forEach(adjacency -> {
2136                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2137                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2138                                 () -> {
2139                                     // TODO Deal with sequencing — the config tx must only submitted
2140                                     // if the oper tx goes in
2141                                     if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2142                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2143                                         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
2144                                                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
2145                                                     confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2146                                                             primaryRd, adjacency, vpnInterfaceOptional.get()
2147                                                                     .getDpnId(), confTx, operTx)))));
2148                                         return futures;
2149                                     } else {
2150                                         return Collections.emptyList();
2151                                     }
2152                                 });
2153                         });
2154             } catch (ReadFailedException e) {
2155                 LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2156                         vpnName, primaryRd);
2157             }
2158         }));
2159     }
2160
2161     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2162         private final String interfaceName;
2163         private final boolean add;
2164         private final String txnDestination;
2165
2166         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2167             this.interfaceName = interfaceName;
2168             this.add = add;
2169             this.txnDestination = transactionDest;
2170         }
2171
2172         @Override
2173         public void onSuccess(Void voidObj) {
2174             if (add) {
2175                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2176                         interfaceName, txnDestination);
2177             } else {
2178                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2179             }
2180         }
2181
2182         @Override
2183         public void onFailure(Throwable throwable) {
2184             if (add) {
2185                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2186                         interfaceName, txnDestination, throwable);
2187             } else {
2188                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2189                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
2190             }
2191         }
2192     }
2193 }