vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  44 import org.opendaylight.genius.mdsalutil.NWUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.infrautils.caches.CacheProvider;
  49 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  50 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  52 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  58 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  59 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  60 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  61 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 108 import org.slf4j.Logger;
 109 import org.slf4j.LoggerFactory;
 110
 111 @Singleton
 112 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 113
 114     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 115     private static final short DJC_MAX_RETRIES = 3;
 116
 117     private final DataBroker dataBroker;
 118     private final ManagedNewTransactionRunner txRunner;
 119     private final IBgpManager bgpManager;
 120     private final IFibManager fibManager;
 121     private final IMdsalApiManager mdsalManager;
 122     private final IdManagerService idManager;
 123     private final OdlInterfaceRpcService ifaceMgrRpcService;
 124     private final VpnFootprintService vpnFootprintService;
 125     private final IInterfaceManager interfaceManager;
 126     private final IVpnManager vpnManager;
 127     private final ArpResponderHandler arpResponderHandler;
 128     private final JobCoordinator jobCoordinator;
 129
 130     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 131
 132     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 133             new ConcurrentHashMap<>();
 134
 135     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 136
 137     @Inject
 138     public VpnInterfaceManager(final DataBroker dataBroker,
 139                                final IBgpManager bgpManager,
 140                                final IdManagerService idManager,
 141                                final IMdsalApiManager mdsalManager,
 142                                final IFibManager fibManager,
 143                                final OdlInterfaceRpcService ifaceMgrRpcService,
 144                                final VpnFootprintService vpnFootprintService,
 145                                final IInterfaceManager interfaceManager,
 146                                final IVpnManager vpnManager,
 147                                final ArpResponderHandler arpResponderHandler,
 148                                final JobCoordinator jobCoordinator,
 149                                final CacheProvider cacheProvider) {
 150         super(VpnInterface.class, VpnInterfaceManager.class);
 151
 152         this.dataBroker = dataBroker;
 153         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 154         this.bgpManager = bgpManager;
 155         this.idManager = idManager;
 156         this.mdsalManager = mdsalManager;
 157         this.fibManager = fibManager;
 158         this.ifaceMgrRpcService = ifaceMgrRpcService;
 159         this.vpnFootprintService = vpnFootprintService;
 160         this.interfaceManager = interfaceManager;
 161         this.vpnManager = vpnManager;
 162         this.arpResponderHandler = arpResponderHandler;
 163         this.jobCoordinator = jobCoordinator;
 164
 165         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 166                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 167                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 168     }
 169
 170     public Runnable isNotifyTaskQueued(String intfName) {
 171         return vpnIntfMap.remove(intfName);
 172     }
 173
 174     @PostConstruct
 175     public void start() {
 176         LOG.info("{} start", getClass().getSimpleName());
 177         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 178     }
 179
 180     @Override
 181     @PreDestroy
 182     public void close() {
 183         super.close();
 184         vpnInstanceOpDataEntryCache.close();
 185     }
 186
 187     @Override
 188     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 189         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 190     }
 191
 192     @Override
 193     protected VpnInterfaceManager getDataTreeChangeListener() {
 194         return VpnInterfaceManager.this;
 195     }
 196
 197     @Override
 198     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 199         LOG.info("add: intfName {} onto vpnName {}",
 200                  vpnInterface.getName(),
 201                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 202         addVpnInterface(identifier, vpnInterface, null, null);
 203     }
 204
 205     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 206                           final VpnInterface vpnInterface, String vpnName) {
 207         synchronized (vpnName.intern()) {
 208             if (isVpnInstanceReady(vpnName)) {
 209                 return true;
 210             }
 211             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 212             return false;
 213         }
 214     }
 215
 216     // TODO Clean up the exception handling
 217     @SuppressWarnings("checkstyle:IllegalCatch")
 218     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 219                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 220         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 221             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 222             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 223         }
 224     }
 225
 226     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 227                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 228         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 229         final String interfaceName = key.getName();
 230
 231         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 232             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 233                   vpnName, vpnInterface.getName());
 234             return;
 235         }
 236         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 237              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 238         List<Adjacency> copyOldAdjs = null;
 239         if (oldAdjs != null) {
 240             copyOldAdjs = new ArrayList<>();
 241             copyOldAdjs.addAll(oldAdjs);
 242         }
 243         List<Adjacency> copyNewAdjs = null;
 244         if (newAdjs != null) {
 245             copyNewAdjs = new ArrayList<>();
 246             copyNewAdjs.addAll(newAdjs);
 247         }
 248         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 249     }
 250
 251     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 252                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 253                     final List<Adjacency> newAdjs,
 254                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 255         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 256         final String interfaceName = key.getName();
 257         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 258         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 259             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 260             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 261             if (interfaceState != null) {
 262                 try {
 263                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 264                     final int ifIndex = interfaceState.getIfIndex();
 265                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 266                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 267                         // (the inventory tx goes in last)
 268                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 269                         ListenableFuture<Void> confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 270                             confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 271                                 operTx -> futures.add(
 272                                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> {
 273                                             LOG.info(
 274                                                     "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 275                                                     + " on dpn {}",
 276                                                     vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 277                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 278                                                     confTx, operTx, invTx, interfaceState, vpnName);
 279                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 280                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 281                                                         + " Update for swapping VPN {} case.", interfaceName, vpnName);
 282                                                 if (newAdjs != null) {
 283                                                     for (Adjacency adj : newAdjs) {
 284                                                         if (oldAdjs.contains(adj)) {
 285                                                             oldAdjs.remove(adj);
 286                                                         } else {
 287                                                             if (!isBgpVpnInternetVpn
 288                                                                     || VpnUtil.isAdjacencyEligibleToVpnInternet(
 289                                                                     dataBroker, adj)) {
 290                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 291                                                                         primaryRd, adj, dpnId, operTx, confTx);
 292                                                             }
 293                                                         }
 294                                                     }
 295                                                 }
 296                                                 for (Adjacency adj : oldAdjs) {
 297                                                     if (!isBgpVpnInternetVpn
 298                                                             || VpnUtil.isAdjacencyEligibleToVpnInternet(
 299                                                             dataBroker, adj)) {
 300                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 301                                                                 operTx, confTx);
 302                                                     }
 303                                                 }
 304                                             }
 305                                         })))));
 306                         futures.add(confFuture);
 307                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 308                                 MoreExecutors.directExecutor());
 309                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 310                                 + " processed successfully", interfaceName, vpnName, dpnId);
 311                         return futures;
 312                     });
 313                 } catch (NumberFormatException | IllegalStateException e) {
 314                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 315                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 316                                     vpnName, e);
 317                     return;
 318                 }
 319             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 320                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 321                     () -> {
 322                         ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
 323                             createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 324                                     confTx, vpnName);
 325                             LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 326                                     vpnName, vpnInterface.getDpnId());
 327                         });
 328                         ListenableFutures.addErrorLogging(future, LOG,
 329                                 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 330                         return Collections.singletonList(future);
 331                     });
 332             } else {
 333                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 334                         + " is not available", interfaceName, vpnName);
 335             }
 336         } else {
 337             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 338                       + " as vpn is pending delete", interfaceName, vpnName,
 339                     vpnInterface.getDpnId());
 340         }
 341     }
 342
 343     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 344     // see comments below.
 345     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 346     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 347             final int lportTag, boolean isInterfaceUp,
 348             WriteTransaction writeConfigTxn,
 349             WriteTransaction writeOperTxn,
 350             WriteTransaction writeInvTxn,
 351             Interface interfaceState,
 352             final String vpnName) {
 353         final String interfaceName = vpnInterface.getName();
 354         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 355                                                          interfaceName, vpnName);
 356         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 357         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 358         if (!isInterfaceUp) {
 359             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 360                      interfaceName, dpId, vpnName);
 361             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 362             if (vpnId == VpnConstants.INVALID_ID) {
 363                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 364                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 365                         dpId);
 366                 return;
 367             }
 368
 369             boolean waitForVpnInterfaceOpRemoval = false;
 370             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 371                 String opVpnName = opVpnInterface.getVpnInstanceName();
 372                 String primaryInterfaceIp = null;
 373                 if (opVpnName.equals(vpnName)) {
 374                     // Please check if the primary VRF Entry does not exist for VPNInterface
 375                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 376                     // back to back
 377                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 378                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 379                     if (adjs == null) {
 380                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 381                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 382                                 vpnName);
 383                         return;
 384                     }
 385                     for (Adjacency adj : adjs) {
 386                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 387                             primaryInterfaceIp = adj.getIpAddress();
 388                             break;
 389                         }
 390                     }
 391                     if (primaryInterfaceIp == null) {
 392                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 393                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 394                                 dpId, vpnName);
 395                         return;
 396                     }
 397                     // Get the rd of the vpn instance
 398                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 399                     if (vrf != null) {
 400                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 401                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 402                         return;
 403                     }
 404                     waitForVpnInterfaceOpRemoval = true;
 405                 } else {
 406                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 407                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 408                 }
 409             }
 410             if (!waitForVpnInterfaceOpRemoval) {
 411                 // Add the VPNInterface and quit
 412                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 413                         null/*ipAddressSourceValuePair*/,
 414                         true /* add */);
 415                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 416                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 417                 if (!isBgpVpnInternetVpn) {
 418                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 419                             jobCoordinator);
 420                 }
 421                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 422                         dpId, vpnName);
 423                 if (interfaceManager.isExternalInterface(interfaceName)) {
 424                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 425                         NwConstants.ADD_FLOW);
 426                 }
 427                 return;
 428             }
 429
 430             // FIB didn't get a chance yet to clean up this VPNInterface
 431             // Let us give it a chance here !
 432             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 433                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 434             try {
 435                 Runnable notifyTask = new VpnNotifyTask();
 436                 synchronized (notifyTask) {
 437                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 438                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 439                     // published here while holding the lock on it so this path will hit the wait before notify can be
 440                     // invoked.
 441                     vpnIntfMap.put(interfaceName, notifyTask);
 442                     try {
 443                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 444                     } catch (InterruptedException e) {
 445                         // Ignored
 446                     }
 447                 }
 448             } finally {
 449                 vpnIntfMap.remove(interfaceName);
 450             }
 451
 452             if (opVpnInterface != null) {
 453                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 454                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 455                         dpId, vpnName);
 456                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
 457                 return;
 458             }
 459             // VPNInterface got removed, proceed with Add
 460             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 461                     interfaceName, dpId, vpnName);
 462             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 463                     null/*ipAddressSourceValuePair*/,
 464                     true /* add */);
 465             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 466                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 467             if (!isBgpVpnInternetVpn) {
 468                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 469             }
 470             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 471                     + " FIB to clean up", interfaceName, dpId, vpnName);
 472             if (interfaceManager.isExternalInterface(interfaceName)) {
 473                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 474                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 475             }
 476         } else {
 477             try {
 478                 // Interface is retained in the DPN, but its Link Up.
 479                 // Advertise prefixes again for this interface to BGP
 480                 InstanceIdentifier<VpnInterface> identifier =
 481                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 482                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 483                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 484                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 485                 // Perform similar operation as interface add event for extraroutes.
 486                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 487                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 488                         LogicalDatastoreType.CONFIGURATION, path);
 489                 if (!optAdjacencies.isPresent()) {
 490                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 491                     return;
 492                 }
 493                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 494                 for (Adjacency adjacency : adjacencies) {
 495                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 496                         continue;
 497                     }
 498                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 499                     if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 500                             dataBroker, adjacency)) {
 501                         continue;
 502                     }
 503                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 504                             dpId, writeOperTxn, writeConfigTxn);
 505                 }
 506             } catch (ReadFailedException e) {
 507                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 508                         interfaceName, vpnName, primaryRd, dpId);
 509             }
 510         }
 511     }
 512
 513     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 514             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 515         Uuid extNetworkId;
 516         try {
 517             // vpn instance of ext-net interface is the network-id
 518             extNetworkId = new Uuid(vpnName);
 519         } catch (IllegalArgumentException e) {
 520             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 521                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 522             return;
 523         }
 524
 525         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 526         if (routerIds == null || routerIds.isEmpty()) {
 527             LOG.info("processExternalVpnInterface: No router is associated with {}."
 528                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 529                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 530             return;
 531         }
 532
 533         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 534                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 535         for (Uuid routerId : routerIds) {
 536             String routerName = routerId.getValue();
 537             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 538             if (Objects.equals(primarySwitch, dpId)) {
 539                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 540                 if (router != null) {
 541                     if (addOrRemove == NwConstants.ADD_FLOW) {
 542                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 543                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 544                                 dpId, interfaceName, lportTag, writeInvTxn);
 545                     } else {
 546                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 547                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 548                                 dpId, interfaceName, lportTag);
 549                     }
 550                 } else {
 551                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 552                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 553                             interfaceName, dpId, vpnName);
 554                 }
 555             }
 556         }
 557     }
 558
 559     // TODO Clean up the exception handling
 560     @SuppressWarnings("checkstyle:IllegalCatch")
 561     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 562                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 563                                                   String vpnName, String interfaceName) {
 564         if (rd == null) {
 565             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 566                   interfaceName, dpnId, vpnName);
 567             return;
 568         } else {
 569             if (rd.equals(vpnName)) {
 570                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 571                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 572                 return;
 573             }
 574         }
 575         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 576                 interfaceName, dpnId, vpnName, rd);
 577
 578         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 579         if (nextHopIp == null) {
 580             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 581                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 582                     rd, vpnName);
 583             return;
 584         }
 585
 586         try {
 587             //Read NextHops
 588             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 589             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 590                     LogicalDatastoreType.OPERATIONAL, path);
 591             if (adjacencies.isPresent()) {
 592                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 593                 if (!nextHops.isEmpty()) {
 594                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 595                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 596                     VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 597                     long l3vni = vpnInstanceOpData.getL3vni();
 598                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 599                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 600                     for (Adjacency nextHop : nextHops) {
 601                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 602                             continue;
 603                         }
 604                         String gatewayMac = null;
 605                         long label = 0;
 606                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 607                             final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 608                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 609                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 610                                     .get();
 611                         } else {
 612                             label = nextHop.getLabel();
 613                         }
 614                         try {
 615                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 616                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 617                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 618                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 619                                     gatewayMac);
 620                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 621                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 622                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 623                         } catch (Exception e) {
 624                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 625                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 626                                     interfaceName, dpnId, e);
 627                         }
 628                     }
 629                 }
 630             }
 631         } catch (ReadFailedException e) {
 632             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 633                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 634         }
 635     }
 636
 637     // TODO Clean up the exception handling
 638     @SuppressWarnings("checkstyle:IllegalCatch")
 639     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 640                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 641         //Read NextHops
 642         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 643         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 644         if (rd == null) {
 645             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 646                 interfaceName, vpnName);
 647             return;
 648         } else {
 649             if (rd.equals(vpnName)) {
 650                 LOG.info(
 651                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 652                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 653                 return;
 654             }
 655         }
 656         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 657                vpnName, rd);
 658         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 659         try {
 660             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 661                     path);
 662         } catch (ReadFailedException e) {
 663             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 664                     interfaceName, vpnName);
 665         }
 666         if (adjacencies.isPresent()) {
 667             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 668
 669             if (!nextHops.isEmpty()) {
 670                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 671                         nextHops, interfaceName, vpnName, rd);
 672                 for (Adjacency nextHop : nextHops) {
 673                     try {
 674                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 675                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 676                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 677                                     interfaceName, vpnName);
 678                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 679                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 680                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 681                                     interfaceName, vpnName);
 682                         } else {
 683                             // Perform similar operation as interface delete event for extraroutes.
 684                             String allocatedRd = nextHop.getVrfId();
 685                             for (String nh : nextHop.getNextHopIpList()) {
 686                                 deleteExtraRouteFromCurrentAndImportingVpns(
 687                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 688                             }
 689                         }
 690                     } catch (Exception e) {
 691                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 692                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 693                     }
 694                 }
 695             }
 696         }
 697     }
 698
 699     @SuppressWarnings("checkstyle:IllegalCatch")
 700     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 701                                                   String primaryRd, String interfaceName, final long vpnId,
 702                                                   WriteTransaction writeConfigTxn,
 703                                                   WriteTransaction writeOperTxn,
 704                                                   final WriteTransaction writeInvTxn,
 705                                                   Interface interfaceState) {
 706         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 707         // Read NextHops
 708         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 709         Optional<Adjacencies> adjacencies = Optional.absent();
 710         try {
 711             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 712                     path);
 713         } catch (ReadFailedException e) {
 714             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 715                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 716         }
 717         if (!adjacencies.isPresent()) {
 718             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 719                     null/*gwMac*/, writeOperTxn);
 720             return;
 721         }
 722         // Get the rd of the vpn instance
 723         String nextHopIp = null;
 724         try {
 725             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 726         } catch (Exception e) {
 727             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 728                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 729         }
 730         List<String> nhList = new ArrayList<>();
 731         if (nextHopIp != null) {
 732             nhList.add(nextHopIp);
 733             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 734                     interfaceName, dpnId, vpnName, nhList);
 735         }
 736         Optional<String> gwMac = Optional.absent();
 737         String vpnInterfaceSubnetGwMacAddress = null;
 738         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 739         Long l3vni = vpnInstanceOpData.getL3vni();
 740         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 741         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 742         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 743         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 744         List<Adjacency> value = new ArrayList<>();
 745         for (Adjacency nextHop : nextHops) {
 746             String rd = primaryRd;
 747             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 748             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 749                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 750                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 751                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 752                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 753                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 754                 continue;
 755             }
 756             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 757                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 758                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 759                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 760                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 761                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 762                 writeOperTxn.merge(
 763                     LogicalDatastoreType.OPERATIONAL,
 764                     VpnUtil.getPrefixToInterfaceIdentifier(
 765                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 766                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 767                             prefixCue), true);
 768                 final Uuid subnetId = nextHop.getSubnetId();
 769
 770                 String gatewayIp = nextHop.getSubnetGatewayIp();
 771                 if (gatewayIp == null) {
 772                     Optional<String> gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 773                     if (gatewayIpOptional.isPresent()) {
 774                         gatewayIp = gatewayIpOptional.get();
 775                     }
 776                 }
 777
 778                 if (gatewayIp != null) {
 779                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 780                     if (gwMac.isPresent()) {
 781                         // A valid mac-address is available for this subnet-gateway-ip
 782                         // Use this for programming ARP_RESPONDER table here.  And save this
 783                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 784                         // to populate L3_GW_MAC_TABLE there.
 785                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 786                                 gatewayIp, gwMac.get());
 787                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 788                     } else {
 789                         // A valid mac-address is not available for this subnet-gateway-ip
 790                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 791                         // Save this connected-mac-address as gateway-mac-address for the
 792                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 793                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 794                         if (gwMac.isPresent()) {
 795                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 796                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 797                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 798                                     gatewayIp, gwMac.get());
 799                         } else {
 800                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 801                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 802                                 + "gwIp {}",
 803                                 subnetId, interfaceName, vpnName, gatewayIp);
 804                         }
 805                     }
 806                 } else {
 807                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 808                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 809                         subnetId, interfaceName, vpnName);
 810                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 811                 }
 812                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 813                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 814             } else {
 815                 //Extra route adjacency
 816                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 817                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 818                 synchronized (vpnPrefixKey.intern()) {
 819                     java.util.Optional<String> rdToAllocate = VpnUtil
 820                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 821                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 822                     if (rdToAllocate.isPresent()) {
 823                         rd = rdToAllocate.get();
 824                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 825                             rd, prefix);
 826                     } else {
 827                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 828                         continue;
 829                     }
 830                 }
 831                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 832                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 833                         interfaceName, dpnId);
 834             }
 835             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 836             // can be different from the gateway-mac-address within the VRFEntry as the
 837             // gateway-mac-address is a superset.
 838             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 839                     : RouteOrigin.STATIC;
 840             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 841                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 842                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 843             Adjacency operationalAdjacency = null;
 844             try {
 845                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 846             } catch (NullPointerException e) {
 847                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 848                     input, e.getMessage());
 849                 return;
 850             }
 851             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 852                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 853                         vpnName, l3vni, origin,
 854                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 855             }
 856             value.add(operationalAdjacency);
 857         }
 858
 859         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 860         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 861                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 862
 863         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 864                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 865                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 866
 867         for (Adjacency nextHop : aug.getAdjacency()) {
 868             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 869             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 870                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 871                         : RouteOrigin.STATIC;
 872                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 873                 registeredPopulator.populateFib(input, writeConfigTxn);
 874             }
 875         }
 876     }
 877
 878     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 879                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 880         VpnInterfaceOpDataEntry opInterface =
 881               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 882         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 883             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 884         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 885                 WriteTransaction.CREATE_MISSING_PARENTS);
 886         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 887                 interfaceName, dpnId, vpnName);
 888     }
 889
 890     // TODO Clean up the exception handling
 891     @SuppressWarnings("checkstyle:IllegalCatch")
 892     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 893                                            StateTunnelList stateTunnelList,
 894                                            WriteTransaction writeConfigTxn,
 895                                            WriteTransaction writeOperTxn) {
 896
 897         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 898         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 899         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 900         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 901         if (adjList.isEmpty()) {
 902             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 903                     vpnInterface, srcDpnId);
 904             return;
 905         }
 906         String prefix = null;
 907         long label = 0;
 908         List<Adjacency> value = new ArrayList<>();
 909         boolean isNextHopAddReqd = false;
 910         String vpnName = vpnInterface.getVpnInstanceName();
 911         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 912         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 913         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 914                 vpnInterface.getName(), vpnInterface.getDpnId(),
 915                 vpnInterface.getVpnInstanceName(), adjList);
 916         for (Adjacency adj : adjList) {
 917             String rd = adj.getVrfId();
 918             rd = rd != null ? rd : vpnName;
 919             prefix = adj.getIpAddress();
 920             label = adj.getLabel();
 921             List<String> nhList = Collections.singletonList(srcTepIp);
 922             List<String> nextHopList = adj.getNextHopIpList();
 923             // If TEP is added , update the nexthop of primary adjacency.
 924             // Secondary adj nexthop is already pointing to primary adj IP address.
 925             if (nextHopList != null && !nextHopList.isEmpty()) {
 926                 /* everything right already */
 927             } else {
 928                 isNextHopAddReqd = true;
 929             }
 930
 931             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 932                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 933             } else {
 934                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 935                 if (!vrfEntryOptional.isPresent()) {
 936                     continue;
 937                 }
 938                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 939                 if (!nhList.contains(srcTepIp)) {
 940                     nhList.add(srcTepIp);
 941                     isNextHopAddReqd = true;
 942                 }
 943                 value.add(adj);
 944             }
 945
 946             if (isNextHopAddReqd) {
 947                 updateLabelMapper(label, nhList);
 948                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 949                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 950                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 951                 // Update the VRF entry with nextHop
 952                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 953                         label, true, writeConfigTxn);
 954
 955                 //Get the list of VPN's importing this route(prefix) .
 956                 // Then update the VRF entry with nhList
 957                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 958                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 959                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 960                     String vpnRd = vpn.getVrfId();
 961                     if (vpnRd != null) {
 962                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 963                                 srcTepIp, label, true, writeConfigTxn);
 964                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 965                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 966                             vpnInterface.getName(), srcDpnId, vpnName,
 967                             vpn.getVpnInstanceName(), vpnRd);
 968                     }
 969                 }
 970                 // Advertise the prefix to BGP only for external vpn
 971                 // since there is a nexthop change.
 972                 try {
 973                     if (!rd.equalsIgnoreCase(vpnName)) {
 974                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 975                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 976                                 null /*gatewayMacAddress*/);
 977                     }
 978                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 979                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 980                             srcDpnId, vpnName);
 981                 } catch (Exception ex) {
 982                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 983                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 984                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 985                 }
 986             }
 987         }
 988         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 989         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 990                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 991                 .addAugmentation(AdjacenciesOp.class, aug).build();
 992         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 993                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 994         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 995                 WriteTransaction.CREATE_MISSING_PARENTS);
 996         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 997                 vpnInterface.getName(), srcDpnId, vpnName);
 998
 999     }
1000
1001     // TODO Clean up the exception handling
1002     @SuppressWarnings("checkstyle:IllegalCatch")
1003     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1004                                               StateTunnelList stateTunnelList,
1005                                               WriteTransaction writeConfigTxn,
1006                                               WriteTransaction writeOperTxn) {
1007
1008         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1009         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1010         String prefix = null;
1011         long label = 0;
1012         boolean isNextHopRemoveReqd = false;
1013         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1014         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1015         String vpnName = vpnInterface.getVpnInstanceName();
1016         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1017         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1018         if (adjList != null) {
1019             List<Adjacency> value = new ArrayList<>();
1020             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1021                     vpnInterface.getName(), vpnInterface.getDpnId(),
1022                     vpnInterface.getVpnInstanceName(), adjList);
1023             for (Adjacency adj : adjList) {
1024                 List<String> nhList = new ArrayList<>();
1025                 String rd = adj.getVrfId();
1026                 rd = rd != null ? rd : vpnName;
1027                 prefix = adj.getIpAddress();
1028                 List<String> nextHopList = adj.getNextHopIpList();
1029                 label = adj.getLabel();
1030                 if (nextHopList != null && !nextHopList.isEmpty()) {
1031                     isNextHopRemoveReqd = true;
1032                 }
1033                 // If TEP is deleted , remove the nexthop from primary adjacency.
1034                 // Secondary adj nexthop will continue to point to primary adj IP address.
1035                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1036                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1037                 } else {
1038                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1039                     if (!vrfEntryOptional.isPresent()) {
1040                         continue;
1041                     }
1042                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1043                     if (nhList.contains(srcTepIp)) {
1044                         nhList.remove(srcTepIp);
1045                         isNextHopRemoveReqd = true;
1046                     }
1047                     value.add(adj);
1048                 }
1049
1050                 if (isNextHopRemoveReqd) {
1051                     updateLabelMapper(label, nhList);
1052                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1053                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1054                             prefix, nhList, vpnName,
1055                             vpnId, rd, vpnInterface.getName());
1056                     // Update the VRF entry with removed nextHop
1057                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1058                             label, false, writeConfigTxn);
1059
1060                     //Get the list of VPN's importing this route(prefix) .
1061                     // Then update the VRF entry with nhList
1062                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1063                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1064                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1065                         String vpnRd = vpn.getVrfId();
1066                         if (vpnRd != null) {
1067                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1068                                     srcTepIp, label, false, writeConfigTxn);
1069                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1070                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1071                                     nhList, label, vpnInterface.getName(), srcDpnId,
1072                                     vpnName,
1073                                     vpn.getVpnInstanceName(), vpnRd);
1074                         }
1075                     }
1076
1077                     // Withdraw prefix from BGP only for external vpn.
1078                     try {
1079                         if (!rd.equalsIgnoreCase(vpnName)) {
1080                             bgpManager.withdrawPrefix(rd, prefix);
1081                         }
1082                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1083                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1084                                 vpnInterface.getName(), srcDpnId,
1085                                 vpnName);
1086                     } catch (Exception ex) {
1087                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1088                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1089                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1090                     }
1091                 }
1092             }
1093             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1094             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1095                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1096                     .addAugmentation(AdjacenciesOp.class, aug).build();
1097             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1098                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1099             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1100                     WriteTransaction.CREATE_MISSING_PARENTS);
1101             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1102                          vpnInterface.getName(), srcDpnId, vpnName);
1103         }
1104     }
1105
1106     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1107         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1108
1109         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1110         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1111         if (vpnInstanceOpDataEntry == null) {
1112             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1113                     + " to check for vpns exporting the routes", vpnName);
1114             return vpnsToExportRoute;
1115         }
1116
1117         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1118             if (input.getVpnInstanceName() == null) {
1119                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1120                         input.getVrfId());
1121                 return false;
1122             }
1123             return !input.getVpnInstanceName().equals(vpnName);
1124         };
1125
1126         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1127             Iterable<String> commonRTs =
1128                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1129                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1130             return Iterators.size(commonRTs.iterator()) > 0;
1131         };
1132
1133         vpnsToExportRoute =
1134                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1135                         Collectors.toList());
1136         return vpnsToExportRoute;
1137     }
1138
1139     // TODO Clean up the exception handling
1140     @SuppressWarnings("checkstyle:IllegalCatch")
1141     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1142         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1143         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1144             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1145             if (vrfEntries != null) {
1146                 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1147                     for (VrfEntry vrfEntry : vrfEntries) {
1148                         try {
1149                             if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1150                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1151                                 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1152                                                 + " is not a controller managed non intervpn link route. Ignoring.",
1153                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1154                                 continue;
1155                             }
1156                             String prefix = vrfEntry.getDestPrefix();
1157                             String gwMac = vrfEntry.getGatewayMacAddress();
1158                             vrfEntry.getRoutePaths().forEach(routePath -> {
1159                                 String nh = routePath.getNexthopAddress();
1160                                 int label = routePath.getLabel().intValue();
1161                                 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1162                                         vrfEntry.getOrigin()))) {
1163                                     LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1164                                                     + " nexthop {} label {} to vpn {} vpnRd {}",
1165                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1166                                     fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1167                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1168                                             0 /*l3vni*/, gwMac,  vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1169                                             confTx);
1170                                 } else {
1171                                     LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1172                                                     + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1173                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1174                                     SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1175                                     importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1176                                             confTx);
1177                                 }
1178                             });
1179                         } catch (RuntimeException e) {
1180                             LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1181                                             + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1182                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1183                         }
1184                     }
1185                 }), LOG, "Error handing VPN exporting routes");
1186             } else {
1187                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1188                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1189             }
1190         }
1191     }
1192
1193     @Override
1194     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1195         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1196         final String interfaceName = key.getName();
1197         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1198             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1199             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1200         }
1201     }
1202
1203     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1204                                 final VpnInterface vpnInterface, final String vpnName,
1205                                 final String interfaceName) {
1206         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1207             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1208                 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1209                     deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1210                     LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1211                 });
1212                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1213                         vpnInterface.getName(), vpnName);
1214                 return Collections.singletonList(future);
1215             }, DJC_MAX_RETRIES);
1216         } else {
1217             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1218             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1219         }
1220     }
1221
1222     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1223     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1224                                 final VpnInterface vpnInterface, final String vpnName,
1225                                 final String interfaceName, final Interface interfaceState) {
1226         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1227                 vpnName, vpnInterface.getDpnId());
1228         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1229         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1230             () -> {
1231                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1232                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1233                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1234                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1235                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1236                                     interfaceName, vpnName);
1237                             BigInteger dpId = BigInteger.ZERO;
1238                             int ifIndex = 0;
1239                             String gwMacAddress = null;
1240                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1241                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1242                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1243                             try {
1244                                 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1245                                         LogicalDatastoreType.OPERATIONAL, interfaceId);
1246                             } catch (ReadFailedException e) {
1247                                 LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName,
1248                                         vpnName);
1249                                 return;
1250                             }
1251                             if (interfaceState != null) {
1252                                 try {
1253                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1254                                 } catch (NumberFormatException | IllegalStateException e) {
1255                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1256                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1257                                                     + " from vpn interface op data store. ", interfaceName,
1258                                             vpnInterface.getDpnId(), vpnName, e);
1259                                     dpId = BigInteger.ZERO;
1260                                 }
1261                                 ifIndex = interfaceState.getIfIndex();
1262                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1263                             } else {
1264                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1265                                         + " from vpn interface op.", interfaceName);
1266                                 if (optVpnInterface.isPresent()) {
1267                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1268                                     dpId = vpnOpInterface.getDpnId();
1269                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1270                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1271                                 } else {
1272                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1273                                                     + " as interfaceState and vpn interface op is not"
1274                                             + " available", interfaceName, vpnName);
1275                                     return;
1276                                 }
1277                             }
1278                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1279                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1280                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1281                             LOG.info(
1282                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1283                                             + "successfully",
1284                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1285                         })))));
1286                 futures.add(configFuture);
1287                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1288                 return futures;
1289             }, DJC_MAX_RETRIES);
1290     }
1291
1292     protected void processVpnInterfaceDown(BigInteger dpId,
1293                                            String interfaceName,
1294                                            int lportTag,
1295                                            String gwMac,
1296                                            VpnInterfaceOpDataEntry vpnOpInterface,
1297                                            boolean isInterfaceStateDown,
1298                                            WriteTransaction writeConfigTxn,
1299                                            WriteTransaction writeOperTxn,
1300                                            WriteTransaction writeInvTxn) {
1301         if (vpnOpInterface == null) {
1302             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1303                     + " as it is not available in operational data store", interfaceName, dpId);
1304             return;
1305         }
1306         final String vpnName = vpnOpInterface.getVpnInstanceName();
1307         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1308                                                     interfaceName, vpnName);
1309         if (!isInterfaceStateDown) {
1310             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1311             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1312                     null);
1313             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1314             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1315                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1316             if (interfaceManager.isExternalInterface(interfaceName)) {
1317                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1318                         NwConstants.DEL_FLOW);
1319             }
1320             if (!isBgpVpnInternetVpn) {
1321                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1322             }
1323             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1324                     + " successful", interfaceName, dpId, vpnName);
1325         } else {
1326             // Interface is retained in the DPN, but its Link Down.
1327             // Only withdraw the prefixes for this interface from BGP
1328             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1329         }
1330     }
1331
1332     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1333                                           final String vpnName, final long vpnId, String gwMac,
1334                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1335                                           final WriteTransaction writeInvTxn) {
1336         //Read NextHops
1337         try {
1338             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1339                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1340             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1341             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1342                     LogicalDatastoreType.OPERATIONAL, path);
1343             String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1344             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1345                     interfaceName, dpnId, vpnName, primaryRd);
1346             if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1347                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1348                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1349                         interfaceName, dpnId, vpnName, nextHops);
1350                 for (Adjacency nextHop : nextHops) {
1351                     if (nextHop.isPhysNetworkFunc()) {
1352                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1353                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1354                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1355                                 null/*writeCfgTxn*/);
1356                     } else {
1357                         String rd = nextHop.getVrfId();
1358                         List<String> nhList;
1359                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1360                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1361                         } else {
1362                             // This is a primary adjacency
1363                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1364                                     : Collections.emptyList();
1365                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1366                                     interfaceName, writeInvTxn);
1367                         }
1368                         if (!nhList.isEmpty()) {
1369                             if (rd.equals(vpnName)) {
1370                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1371                                 //remove from FIB directly
1372                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1373                                         interfaceName, dpnId, writeConfigTxn));
1374                             } else {
1375                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1376                                         interfaceName, writeConfigTxn);
1377                             }
1378                         } else {
1379                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1380                                             + " interface {}", nextHop.getIpAddress(), rd,
1381                                     nextHop.getAdjacencyType().toString(), interfaceName);
1382                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1383                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1384                         }
1385                     }
1386                     String ip = nextHop.getIpAddress().split("/")[0];
1387                     LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1388                     if (vpnVipToPort != null) {
1389                         VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip, null);
1390                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1391                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1392                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1393                     }
1394                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ip);
1395                     if (vpnPortipToPort != null) {
1396                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1397                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1398                                  + "Interface {} ip {} on dpn {} for vpn {}",
1399                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1400                     }
1401                 }
1402             } else {
1403                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1404                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1405                         + " Removing it.", interfaceName, vpnName, dpnId);
1406                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1407             }
1408         } catch (ReadFailedException e) {
1409             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1410                     interfaceName, dpnId, vpnName);
1411         }
1412     }
1413
1414     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1415                                                                    String interfaceName, BigInteger dpnId,
1416                                                                    WriteTransaction writeConfigTxn) {
1417         return (nh) -> {
1418             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1419                     writeConfigTxn);
1420             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1421                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1422                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1423         };
1424     }
1425
1426     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1427                                            BigInteger dpnId, String rd, String interfaceName,
1428                                            WriteTransaction writeConfigTxn) {
1429         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1430                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1431         nhList.forEach((nh) -> {
1432             //IRT: remove routes from other vpns importing it
1433             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1434                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1435             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1436                 String vpnRd = vpn.getVrfId();
1437                 if (vpnRd != null) {
1438                     fibManager.removeOrUpdateFibEntry(vpnRd,
1439                             nextHop.getIpAddress(), nh, writeConfigTxn);
1440                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1441                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1442                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1443                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1444                 }
1445             }
1446         });
1447     }
1448
1449     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1450                                                  int lportTag, String gwMac, String interfaceName,
1451                                                  WriteTransaction writeInvTxn) {
1452         final Uuid subnetId = nextHop.getSubnetId();
1453         if (nextHop.getSubnetGatewayMacAddress() == null) {
1454             // A valid mac-address was not available for this subnet-gateway-ip
1455             // So a connected-mac-address was used for this subnet and we need
1456             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1457             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1458                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1459         }
1460         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1461                 subnetId);
1462     }
1463
1464     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1465                                                   String interfaceName) {
1466         // This is either an extra-route (or) a learned IP via subnet-route
1467         List<String> nhList = null;
1468         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1469         if (nextHopIp == null || nextHopIp.isEmpty()) {
1470             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1471                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1472                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1473                     vpnName);
1474             nhList = Collections.emptyList();
1475         } else {
1476             nhList = Collections.singletonList(nextHopIp);
1477         }
1478         return nhList;
1479     }
1480
1481     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1482         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1483         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1484         // else use connected interface
1485         if (gwPort != null && gwPort.isSubnetIp()) {
1486             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1487                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1488             return Optional.of(gwPort.getMacAddress());
1489         }
1490         return Optional.absent();
1491     }
1492
1493     @Override
1494     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1495         final VpnInterface update) {
1496         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1497                 update.getDpnId(), original.getVpnInstanceNames(),
1498                 update.getVpnInstanceNames());
1499         final String vpnInterfaceName = update.getName();
1500         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1501         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1502         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1503             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1504         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1505         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1506             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1507
1508         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1509         //handles switching between <internal VPN - external VPN>
1510         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1511             LOG.info("update: handled VPNInterface {} on dpn {} update"
1512                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1513                      original.getName(), dpnId,
1514                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1515                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1516             return;
1517         }
1518         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1519             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1520             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1521             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1522             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1523             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1524                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1525                     // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1526                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1527                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1528                         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(operTx -> {
1529                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1530                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1531                             LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1532                                     update.getName(), newVpnName);
1533                             //handle both addition and removal of adjacencies
1534                             //currently, new adjacency may be an extra route
1535                             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1536                             if (!oldAdjs.equals(newAdjs)) {
1537                                 for (Adjacency adj : copyNewAdjs) {
1538                                     if (copyOldAdjs.contains(adj)) {
1539                                         copyOldAdjs.remove(adj);
1540                                     } else {
1541                                         // add new adjacency - right now only extra route will hit this path
1542                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker,
1543                                                 adj)) {
1544                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1545                                                     dpnId, operTx, confTx);
1546                                         }
1547                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1548                                                 + " vpn interface {} on vpn {} dpnId {}",
1549                                                 adj.getIpAddress(), adj.getNextHopIpList(),
1550                                                 adj.getLabel(), adj.getSubnetId(), update.getName(),
1551                                                 newVpnName, dpnId);
1552                                     }
1553                                 }
1554                                 for (Adjacency adj : copyOldAdjs) {
1555                                     if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker,
1556                                             adj)) {
1557                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1558                                                 && !adj.isPhysNetworkFunc()) {
1559                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1560                                                     operTx, confTx);
1561                                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = operTx.read(
1562                                                     LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier).get();
1563                                             if (optVpnInterface.isPresent()) {
1564                                                 VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get();
1565                                                 long vpnId = VpnUtil.getVpnId(dataBroker, newVpnName);
1566                                                 VpnUtil.removePrefixToInterfaceAdj(dataBroker, adj, vpnId,
1567                                                         vpnInterfaceOpDataEntry, operTx);
1568                                             } else {
1569                                                 LOG.info("Vpninterface {} not present in Operational",
1570                                                         vpnInterfaceName);
1571                                             }
1572                                             //remove FIB entry
1573                                             String vpnRd = VpnUtil.getVpnRd(dataBroker, newVpnName);
1574                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1575                                                     + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1576                                             //remove BGP entry
1577                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1578                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1579                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1580                                             }
1581                                         } else {
1582                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1583                                                     operTx, confTx);
1584                                         }
1585                                     }
1586                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1587                                                     + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1588                                                     .getNextHopIpList(),
1589                                             adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1590                                 }
1591                             }
1592                         }));
1593                     }));
1594                     for (ListenableFuture<Void> future : futures) {
1595                         ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1596                                 update.getName(), update.getVpnInstanceNames());
1597                     }
1598                     return futures;
1599                 });
1600             } else {
1601                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1602                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1603             }
1604         }
1605     }
1606
1607     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1608                                                  VpnInterface original, VpnInterface update) {
1609         boolean isSwap = Boolean.FALSE;
1610         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1611         final String interfaceName = key.getName();
1612         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1613             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1614         List<String> oldVpnListCopy = new ArrayList<>();
1615         oldVpnListCopy.addAll(oldVpnList);
1616         List<String> newVpnList = update.getVpnInstanceNames().stream()
1617             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1618         oldVpnList.removeAll(newVpnList);
1619         newVpnList.removeAll(oldVpnListCopy);
1620         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1621             for (String oldVpnName: oldVpnList) {
1622                 isSwap = Boolean.TRUE;
1623                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1624                         + " running config-driven swap removal", interfaceName, oldVpnName);
1625                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1626                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1627                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1628             }
1629             //Wait for previous interface bindings to be removed
1630             try {
1631                 Thread.sleep(2000);
1632             } catch (InterruptedException e) {
1633                 //Ignore
1634             }
1635             for (String newVpnName: newVpnList) {
1636                 String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1637                 isSwap = Boolean.TRUE;
1638                 if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1639                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1640                             + "running config-driven swap addition", interfaceName, newVpnName);
1641                     final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1642                     final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1643                             ? origAdjs.getAdjacency() : new ArrayList<>();
1644                     final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1645                     final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1646                             ? updateAdjs.getAdjacency() : new ArrayList<>();
1647
1648                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1649                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1650                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1651                             interfaceName, oldVpnListCopy, newVpnName);
1652                 }
1653             }
1654         }
1655         return isSwap;
1656     }
1657
1658     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1659         try {
1660             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1661             synchronized (label.toString().intern()) {
1662                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1663                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1664                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1665                         LogicalDatastoreType.OPERATIONAL, lriIid);
1666                 if (opResult.isPresent()) {
1667                     LabelRouteInfo labelRouteInfo =
1668                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1669                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1670                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1671                 }
1672             }
1673             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1674                     nextHopIpList);
1675         } catch (ReadFailedException e) {
1676             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1677                     nextHopIpList);
1678         } catch (TransactionCommitFailedException e) {
1679             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1680                     nextHopIpList);
1681         }
1682     }
1683
1684     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1685         SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1686
1687         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1688         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1689                 .addAugmentation(SubnetRoute.class, route).build();
1690         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1691         InstanceIdentifierBuilder<VrfTables> idBuilder =
1692             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1693         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1694         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1695         if (writeConfigTxn != null) {
1696             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1697         } else {
1698             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1699         }
1700         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1701                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1702     }
1703
1704     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1705                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1706                                            WriteTransaction writeConfigTxn) {
1707         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1708         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1709         try {
1710             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1711                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1712             if (optVpnInterface.isPresent()) {
1713                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1714                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1715                 String vpnName = currVpnIntf.getVpnInstanceName();
1716                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1717                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1718                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1719                         LogicalDatastoreType.OPERATIONAL, adjPath);
1720                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1721                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1722                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1723                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1724                 List<Adjacency> adjacencies;
1725                 if (optAdjacencies.isPresent()) {
1726                     adjacencies = optAdjacencies.get().getAdjacency();
1727                 } else {
1728                     // This code will be hit in case of first PNF adjacency
1729                     adjacencies = new ArrayList<>();
1730                 }
1731                 long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1732                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1733                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1734                 Adjacency operationalAdjacency = null;
1735                 //Handling dual stack neutron port primary adjacency
1736                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1737                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1738                             currVpnIntf.getName(), vpnName);
1739                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1740                             currVpnIntf.getName());
1741                     if (interfaceState != null) {
1742                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1743                                 currVpnIntf.getName(),
1744                                 vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1745                     }
1746                 }
1747                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1748                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1749                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1750                             : RouteOrigin.STATIC;
1751                     String nh = adj.getNextHopIpList().get(0);
1752                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1753                     synchronized (vpnPrefixKey.intern()) {
1754                         java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1755                                 dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1756                         if (rdToAllocate.isPresent()) {
1757                             input.setRd(rdToAllocate.get());
1758                             operationalAdjacency = populator.createOperationalAdjacency(input);
1759                             int label = operationalAdjacency.getLabel().intValue();
1760                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1761                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1762                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1763                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1764                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1765                                     vpnName, label, currVpnIntf.getName(), dpnId);
1766                         } else {
1767                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1768                                     vpnName, prefix);
1769                             return;
1770                         }
1771                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1772                         // Keeping the MPLS check for now.
1773                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1774                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1775                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1776                                     VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1777                             vpnsToImportRoute.forEach(vpn -> {
1778                                 if (vpn.getVrfId() != null) {
1779                                     VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1780                                             dataBroker, vpn.getVpnId(), vpnId, prefix,
1781                                             VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1782                                             .ifPresent(
1783                                                 rds -> vpnManager.addExtraRoute(
1784                                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1785                                                         adj.getIpAddress(), nh, rds,
1786                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1787                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1788                                                         opAdjacency, encapType, writeConfigTxn));
1789                                 }
1790                             });
1791                         }
1792                     }
1793                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1794                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1795                             currVpnIntf.getName(), vpnName);
1796
1797                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1798
1799                     writeOperTxn.merge(
1800                             LogicalDatastoreType.OPERATIONAL,
1801                             VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1802                                     adj.getSubnetId().getValue()), prefix),
1803                             VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1804                                     adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1805
1806                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1807                             adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1808                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1809
1810                     input.setRd(adj.getVrfId());
1811                 }
1812                 if (operationalAdjacency == null) {
1813                     operationalAdjacency = populator.createOperationalAdjacency(input);
1814                 }
1815                 adjacencies.add(operationalAdjacency);
1816                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1817                 VpnInterfaceOpDataEntry newVpnIntf =
1818                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1819                                 aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1820                                 currVpnIntf.getGatewayMacAddress());
1821
1822                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1823             }
1824         } catch (ReadFailedException e) {
1825             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1826                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1827         }
1828     }
1829
1830     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1831         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1832         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1833     }
1834
1835     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1836             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1837         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1838         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1839         try {
1840             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1841                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1842             if (optVpnInterface.isPresent()) {
1843                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1844                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1845                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1846                         LogicalDatastoreType.OPERATIONAL, path);
1847                 if (optAdjacencies.isPresent()) {
1848                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1849
1850                     if (!adjacencies.isEmpty()) {
1851                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1852                         Iterator<Adjacency> adjIt = adjacencies.iterator();
1853                         while (adjIt.hasNext()) {
1854                             Adjacency adjElem = adjIt.next();
1855                             if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1856                                 String rd = adjElem.getVrfId();
1857                                 adjIt.remove();
1858                                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1859                                 VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1860                                         .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1861                                                 currVpnIntf.getVpnInstanceName(), aug, dpnId,
1862                                                 currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1863                                                 currVpnIntf.getGatewayMacAddress());
1864                                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1865                                 if (adj.getNextHopIpList() != null) {
1866                                     for (String nh : adj.getNextHopIpList()) {
1867                                         deleteExtraRouteFromCurrentAndImportingVpns(
1868                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1869                                                 currVpnIntf.getName(), writeConfigTxn);
1870                                     }
1871                                 } else if (adj.isPhysNetworkFunc()) {
1872                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1873                                             adj.getIpAddress(), adj.getSubnetId());
1874                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1875                                             writeConfigTxn);
1876                                 }
1877                                 break;
1878                             }
1879
1880                         }
1881                     }
1882                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1883                             dpnId, adj.getVrfId());
1884                 } else {
1885                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1886                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1887                 }
1888             }
1889         } catch (ReadFailedException e) {
1890             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1891                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1892         }
1893     }
1894
1895     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1896         String rd, String intfName, WriteTransaction writeConfigTxn) {
1897         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1898         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1899         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1900             String vpnRd = vpn.getVrfId();
1901             if (vpnRd != null) {
1902                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1903             }
1904         }
1905     }
1906
1907     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1908         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1909             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1910             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1911     }
1912
1913     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1914         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1915             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1916     }
1917
1918     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1919                                                     WriteTransaction writeConfigTxn, String vpnName) {
1920         if (vpnInterface == null) {
1921             return;
1922         }
1923         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1924         if (adjs == null) {
1925             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1926                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1927             return;
1928         }
1929         for (Adjacency adj : adjs) {
1930             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1931                 String primaryInterfaceIp = adj.getIpAddress();
1932                 String macAddress = adj.getMacAddress();
1933                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1934
1935                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1936                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1937
1938                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1939                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1940                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1941                         routerInt, label, writeConfigTxn);
1942                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1943                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1944                         macAddress);
1945                 return;
1946             }
1947         }
1948         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1949                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1950                 primaryRd, vpnName);
1951     }
1952
1953     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1954                                                     WriteTransaction writeConfigTxn, String vpnName) {
1955         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
1956         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1957         if (adjs != null) {
1958             List<Adjacency> adjsList = adjs.getAdjacency();
1959             for (Adjacency adj : adjsList) {
1960                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1961                     String primaryInterfaceIp = adj.getIpAddress();
1962                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1963                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1964                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1965                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1966                     return;
1967                 }
1968             }
1969         } else {
1970             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1971                     vpnInterface.getName(), rd);
1972         }
1973     }
1974
1975     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1976         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1977             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1978                   vpnName, intefaceData.vpnInterface.getName());
1979             return;
1980         }
1981         final VpnInterfaceKey key = intefaceData.identifier
1982                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1983         final String interfaceName = key.getName();
1984         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1985                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1986         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1987                   intefaceData.identifier, vpnName);
1988         return;
1989     }
1990
1991     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1992                                               VpnInterface vpnInterface, String vpnName) {
1993         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
1994                .get(vpnName);
1995         if (vpnInterfaces == null) {
1996             vpnInterfaces = new ConcurrentLinkedQueue<>();
1997         }
1998         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
1999         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2000         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2001                 vpnInterface.getName(), vpnName);
2002     }
2003
2004     public boolean isVpnInstanceReady(String vpnInstanceName) {
2005         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2006         if (vpnRd == null) {
2007             return false;
2008         }
2009         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2010
2011         return vpnInstanceOpDataEntry != null;
2012     }
2013
2014     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2015         synchronized (vpnInstanceName.intern()) {
2016             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2017                     unprocessedVpnInterfaces.get(vpnInstanceName);
2018             if (vpnInterfaces != null) {
2019                 while (!vpnInterfaces.isEmpty()) {
2020                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2021                     if (hasVpnInstanceCreatedSuccessfully) {
2022                         processSavedInterface(savedInterface, vpnInstanceName);
2023                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2024                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2025                     } else {
2026                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2027                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2028                     }
2029                 }
2030             } else {
2031                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2032             }
2033         }
2034     }
2035
2036     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2037             VpnInterface vpnInterface) {
2038         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2039             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2040                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2041             if (vpnInterfaces != null) {
2042                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2043                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2044                             + "unprocessed list", vpnInterface.getName(),
2045                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2046                 }
2047             } else {
2048                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2049                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2050             }
2051         }
2052     }
2053
2054     public void vpnInstanceIsReady(String vpnInstanceName) {
2055         processSavedInterfaces(vpnInstanceName, true);
2056     }
2057
2058     public void vpnInstanceFailed(String vpnInstanceName) {
2059         processSavedInterfaces(vpnInstanceName, false);
2060     }
2061
2062     private static class UnprocessedVpnInterfaceData {
2063         InstanceIdentifier<VpnInterface> identifier;
2064         VpnInterface vpnInterface;
2065
2066         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2067             this.identifier = identifier;
2068             this.vpnInterface = vpnInterface;
2069         }
2070
2071         @Override
2072         public int hashCode() {
2073             final int prime = 31;
2074             int result = 1;
2075             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2076             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2077             return result;
2078         }
2079
2080         @Override
2081         public boolean equals(Object obj) {
2082             if (this == obj) {
2083                 return true;
2084             }
2085             if (obj == null) {
2086                 return false;
2087             }
2088             if (getClass() != obj.getClass()) {
2089                 return false;
2090             }
2091             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2092             if (identifier == null) {
2093                 if (other.identifier != null) {
2094                     return false;
2095                 }
2096             } else if (!identifier.equals(other.identifier)) {
2097                 return false;
2098             }
2099             if (vpnInterface == null) {
2100                 if (other.vpnInterface != null) {
2101                     return false;
2102                 }
2103             } else if (!vpnInterface.equals(other.vpnInterface)) {
2104                 return false;
2105             }
2106             return true;
2107         }
2108     }
2109
2110     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2111         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2112         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2113         if (vpnInstanceOpData == null) {
2114             return;
2115         }
2116         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2117         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2118             return;
2119         }
2120         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2121         vpnToDpnLists.forEach(vpnToDpnList -> {
2122             if (vpnToDpnList.getVpnInterfaces() == null) {
2123                 return;
2124             }
2125             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2126                 try {
2127                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2128                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2129                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2130                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2131                     if (!vpnInterfaceOptional.isPresent()) {
2132                         return;
2133                     }
2134                     List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2135                             vpnInterface.getInterfaceName());
2136                     if (configVpnAdjacencies == null) {
2137                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2138                         return;
2139                     }
2140                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2141                             .augmentation(AdjacenciesOp.class).getAdjacency();
2142                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2143                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2144                     // in operational DS. These unprocessed adjacencies will be handled below.
2145                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2146                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2147                     configVpnAdjacencies.stream()
2148                         .filter(adjacency -> operationVpnAdjacencies.stream()
2149                                 .noneMatch(operationalAdjacency ->
2150                                         operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2151                         .forEach(adjacency -> {
2152                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2153                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2154                                 () -> {
2155                                     // TODO Deal with sequencing — the config tx must only submitted
2156                                     // if the oper tx goes in
2157                                     if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2158                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2159                                         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
2160                                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
2161                                                 confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2162                                                     primaryRd, adjacency, vpnInterfaceOptional.get()
2163                                                         .getDpnId(), confTx, operTx)))));
2164                                         return futures;
2165                                     } else {
2166                                         return Collections.emptyList();
2167                                     }
2168                                 });
2169                         });
2170                 } catch (ReadFailedException e) {
2171                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2172                             vpnName, primaryRd);
2173                 }
2174             });
2175         });
2176     }
2177
2178     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2179         private final String interfaceName;
2180         private final boolean add;
2181         private final String txnDestination;
2182
2183         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2184             this.interfaceName = interfaceName;
2185             this.add = add;
2186             this.txnDestination = transactionDest;
2187         }
2188
2189         @Override
2190         public void onSuccess(Void voidObj) {
2191             if (add) {
2192                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2193                         interfaceName, txnDestination);
2194             } else {
2195                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2196             }
2197         }
2198
2199         @Override
2200         public void onFailure(Throwable throwable) {
2201             if (add) {
2202                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2203                         interfaceName, txnDestination, throwable);
2204             } else {
2205                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2206                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
2207             }
2208         }
2209     }
2210 }