vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  44 import org.opendaylight.genius.mdsalutil.NWUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.infrautils.caches.CacheProvider;
  49 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  50 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  52 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  58 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  59 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  60 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  61 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 108 import org.slf4j.Logger;
 109 import org.slf4j.LoggerFactory;
 110
 111 @Singleton
 112 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 113
 114     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 115     private static final short DJC_MAX_RETRIES = 3;
 116
 117     private final DataBroker dataBroker;
 118     private final ManagedNewTransactionRunner txRunner;
 119     private final IBgpManager bgpManager;
 120     private final IFibManager fibManager;
 121     private final IMdsalApiManager mdsalManager;
 122     private final IdManagerService idManager;
 123     private final OdlInterfaceRpcService ifaceMgrRpcService;
 124     private final VpnFootprintService vpnFootprintService;
 125     private final IInterfaceManager interfaceManager;
 126     private final IVpnManager vpnManager;
 127     private final ArpResponderHandler arpResponderHandler;
 128     private final JobCoordinator jobCoordinator;
 129
 130     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 131
 132     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 133             new ConcurrentHashMap<>();
 134
 135     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 136
 137     @Inject
 138     public VpnInterfaceManager(final DataBroker dataBroker,
 139                                final IBgpManager bgpManager,
 140                                final IdManagerService idManager,
 141                                final IMdsalApiManager mdsalManager,
 142                                final IFibManager fibManager,
 143                                final OdlInterfaceRpcService ifaceMgrRpcService,
 144                                final VpnFootprintService vpnFootprintService,
 145                                final IInterfaceManager interfaceManager,
 146                                final IVpnManager vpnManager,
 147                                final ArpResponderHandler arpResponderHandler,
 148                                final JobCoordinator jobCoordinator,
 149                                final CacheProvider cacheProvider) {
 150         super(VpnInterface.class, VpnInterfaceManager.class);
 151
 152         this.dataBroker = dataBroker;
 153         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 154         this.bgpManager = bgpManager;
 155         this.idManager = idManager;
 156         this.mdsalManager = mdsalManager;
 157         this.fibManager = fibManager;
 158         this.ifaceMgrRpcService = ifaceMgrRpcService;
 159         this.vpnFootprintService = vpnFootprintService;
 160         this.interfaceManager = interfaceManager;
 161         this.vpnManager = vpnManager;
 162         this.arpResponderHandler = arpResponderHandler;
 163         this.jobCoordinator = jobCoordinator;
 164
 165         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 166                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 167                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 168     }
 169
 170     public Runnable isNotifyTaskQueued(String intfName) {
 171         return vpnIntfMap.remove(intfName);
 172     }
 173
 174     @PostConstruct
 175     public void start() {
 176         LOG.info("{} start", getClass().getSimpleName());
 177         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 178     }
 179
 180     @Override
 181     @PreDestroy
 182     public void close() {
 183         super.close();
 184         vpnInstanceOpDataEntryCache.close();
 185     }
 186
 187     @Override
 188     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 189         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 190     }
 191
 192     @Override
 193     protected VpnInterfaceManager getDataTreeChangeListener() {
 194         return VpnInterfaceManager.this;
 195     }
 196
 197     @Override
 198     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 199         LOG.info("add: intfName {} onto vpnName {}",
 200                  vpnInterface.getName(),
 201                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 202         addVpnInterface(identifier, vpnInterface, null, null);
 203     }
 204
 205     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 206                           final VpnInterface vpnInterface, String vpnName) {
 207         synchronized (vpnName.intern()) {
 208             if (isVpnInstanceReady(vpnName)) {
 209                 return true;
 210             }
 211             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 212             return false;
 213         }
 214     }
 215
 216     // TODO Clean up the exception handling
 217     @SuppressWarnings("checkstyle:IllegalCatch")
 218     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 219                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 220         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 221             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 222             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 223         }
 224     }
 225
 226     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 227                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 228         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 229         final String interfaceName = key.getName();
 230
 231         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 232             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 233                   vpnName, vpnInterface.getName());
 234             return;
 235         }
 236         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 237              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 238         List<Adjacency> copyOldAdjs = null;
 239         if (oldAdjs != null) {
 240             copyOldAdjs = new ArrayList<>();
 241             copyOldAdjs.addAll(oldAdjs);
 242         }
 243         List<Adjacency> copyNewAdjs = null;
 244         if (newAdjs != null) {
 245             copyNewAdjs = new ArrayList<>();
 246             copyNewAdjs.addAll(newAdjs);
 247         }
 248         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 249     }
 250
 251     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 252                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 253                     final List<Adjacency> newAdjs,
 254                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 255         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 256         final String interfaceName = key.getName();
 257         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 258         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 259             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 260             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 261             if (interfaceState != null) {
 262                 try {
 263                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 264                     final int ifIndex = interfaceState.getIfIndex();
 265                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 266                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 267                         // (the inventory tx goes in last)
 268                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 269                         ListenableFuture<Void> confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 270                             confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 271                                 operTx -> futures.add(
 272                                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> {
 273                                             LOG.info(
 274                                                     "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 275                                                     + " on dpn {}",
 276                                                     vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 277                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 278                                                     confTx, operTx, invTx, interfaceState, vpnName);
 279                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 280                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 281                                                         + " Update for swapping VPN {} case.", interfaceName, vpnName);
 282                                                 if (newAdjs != null) {
 283                                                     for (Adjacency adj : newAdjs) {
 284                                                         if (oldAdjs.contains(adj)) {
 285                                                             oldAdjs.remove(adj);
 286                                                         } else {
 287                                                             if (!isBgpVpnInternetVpn
 288                                                                     || VpnUtil.isAdjacencyEligibleToVpnInternet(
 289                                                                     dataBroker, adj)) {
 290                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 291                                                                         primaryRd, adj, dpnId, operTx, confTx);
 292                                                             }
 293                                                         }
 294                                                     }
 295                                                 }
 296                                                 for (Adjacency adj : oldAdjs) {
 297                                                     if (!isBgpVpnInternetVpn
 298                                                             || VpnUtil.isAdjacencyEligibleToVpnInternet(
 299                                                             dataBroker, adj)) {
 300                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 301                                                                 operTx, confTx);
 302                                                     }
 303                                                 }
 304                                             }
 305                                         })))));
 306                         futures.add(confFuture);
 307                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 308                                 MoreExecutors.directExecutor());
 309                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 310                                 + " processed successfully", interfaceName, vpnName, dpnId);
 311                         return futures;
 312                     });
 313                 } catch (NumberFormatException | IllegalStateException e) {
 314                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 315                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 316                                     vpnName, e);
 317                     return;
 318                 }
 319             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 320                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 321                     () -> {
 322                         ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
 323                             createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 324                                     confTx, vpnName);
 325                             LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 326                                     vpnName, vpnInterface.getDpnId());
 327                         });
 328                         ListenableFutures.addErrorLogging(future, LOG,
 329                                 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 330                         return Collections.singletonList(future);
 331                     });
 332             } else {
 333                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 334                         + " is not available", interfaceName, vpnName);
 335             }
 336         } else {
 337             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 338                       + " as vpn is pending delete", interfaceName, vpnName,
 339                     vpnInterface.getDpnId());
 340         }
 341     }
 342
 343     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 344     // see comments below.
 345     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 346     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 347             final int lportTag, boolean isInterfaceUp,
 348             WriteTransaction writeConfigTxn,
 349             WriteTransaction writeOperTxn,
 350             WriteTransaction writeInvTxn,
 351             Interface interfaceState,
 352             final String vpnName) {
 353         final String interfaceName = vpnInterface.getName();
 354         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 355                                                          interfaceName, vpnName);
 356         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 357         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 358         if (!isInterfaceUp) {
 359             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 360                      interfaceName, dpId, vpnName);
 361             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 362             if (vpnId == VpnConstants.INVALID_ID) {
 363                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 364                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 365                         dpId);
 366                 return;
 367             }
 368
 369             boolean waitForVpnInterfaceOpRemoval = false;
 370             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 371                 String opVpnName = opVpnInterface.getVpnInstanceName();
 372                 String primaryInterfaceIp = null;
 373                 if (opVpnName.equals(vpnName)) {
 374                     // Please check if the primary VRF Entry does not exist for VPNInterface
 375                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 376                     // back to back
 377                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 378                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 379                     if (adjs == null) {
 380                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 381                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 382                                 vpnName);
 383                         return;
 384                     }
 385                     for (Adjacency adj : adjs) {
 386                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 387                             primaryInterfaceIp = adj.getIpAddress();
 388                             break;
 389                         }
 390                     }
 391                     if (primaryInterfaceIp == null) {
 392                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 393                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 394                                 dpId, vpnName);
 395                         return;
 396                     }
 397                     // Get the rd of the vpn instance
 398                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 399                     if (vrf != null) {
 400                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 401                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 402                         return;
 403                     }
 404                     waitForVpnInterfaceOpRemoval = true;
 405                 } else {
 406                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 407                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 408                 }
 409             }
 410             if (!waitForVpnInterfaceOpRemoval) {
 411                 // Add the VPNInterface and quit
 412                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 413                         null/*ipAddressSourceValuePair*/,
 414                         true /* add */);
 415                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 416                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 417                 if (!isBgpVpnInternetVpn) {
 418                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 419                             jobCoordinator);
 420                 }
 421                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 422                         dpId, vpnName);
 423                 if (interfaceManager.isExternalInterface(interfaceName)) {
 424                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 425                         NwConstants.ADD_FLOW);
 426                 }
 427                 return;
 428             }
 429
 430             // FIB didn't get a chance yet to clean up this VPNInterface
 431             // Let us give it a chance here !
 432             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 433                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 434             try {
 435                 Runnable notifyTask = new VpnNotifyTask();
 436                 synchronized (notifyTask) {
 437                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 438                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 439                     // published here while holding the lock on it so this path will hit the wait before notify can be
 440                     // invoked.
 441                     vpnIntfMap.put(interfaceName, notifyTask);
 442                     try {
 443                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 444                     } catch (InterruptedException e) {
 445                         // Ignored
 446                     }
 447                 }
 448             } finally {
 449                 vpnIntfMap.remove(interfaceName);
 450             }
 451
 452             if (opVpnInterface != null) {
 453                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 454                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 455                         dpId, vpnName);
 456                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
 457                 return;
 458             }
 459             // VPNInterface got removed, proceed with Add
 460             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 461                     interfaceName, dpId, vpnName);
 462             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 463                     null/*ipAddressSourceValuePair*/,
 464                     true /* add */);
 465             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 466                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 467             if (!isBgpVpnInternetVpn) {
 468                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 469             }
 470             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 471                     + " FIB to clean up", interfaceName, dpId, vpnName);
 472             if (interfaceManager.isExternalInterface(interfaceName)) {
 473                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 474                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 475             }
 476         } else {
 477             try {
 478                 // Interface is retained in the DPN, but its Link Up.
 479                 // Advertise prefixes again for this interface to BGP
 480                 InstanceIdentifier<VpnInterface> identifier =
 481                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 482                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 483                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 484                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 485                 // Perform similar operation as interface add event for extraroutes.
 486                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 487                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 488                         LogicalDatastoreType.CONFIGURATION, path);
 489                 if (!optAdjacencies.isPresent()) {
 490                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 491                     return;
 492                 }
 493                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 494                 for (Adjacency adjacency : adjacencies) {
 495                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 496                         continue;
 497                     }
 498                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 499                     if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 500                             dataBroker, adjacency)) {
 501                         continue;
 502                     }
 503                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 504                             dpId, writeOperTxn, writeConfigTxn);
 505                 }
 506             } catch (ReadFailedException e) {
 507                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 508                         interfaceName, vpnName, primaryRd, dpId);
 509             }
 510         }
 511     }
 512
 513     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 514             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 515         Uuid extNetworkId;
 516         try {
 517             // vpn instance of ext-net interface is the network-id
 518             extNetworkId = new Uuid(vpnName);
 519         } catch (IllegalArgumentException e) {
 520             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 521                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 522             return;
 523         }
 524
 525         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 526         if (routerIds == null || routerIds.isEmpty()) {
 527             LOG.info("processExternalVpnInterface: No router is associated with {}."
 528                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 529                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 530             return;
 531         }
 532
 533         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 534                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 535         for (Uuid routerId : routerIds) {
 536             String routerName = routerId.getValue();
 537             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 538             if (Objects.equals(primarySwitch, dpId)) {
 539                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 540                 if (router != null) {
 541                     if (addOrRemove == NwConstants.ADD_FLOW) {
 542                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 543                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 544                                 dpId, interfaceName, lportTag, writeInvTxn);
 545                     } else {
 546                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 547                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 548                                 dpId, interfaceName, lportTag);
 549                     }
 550                 } else {
 551                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 552                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 553                             interfaceName, dpId, vpnName);
 554                 }
 555             }
 556         }
 557     }
 558
 559     // TODO Clean up the exception handling
 560     @SuppressWarnings("checkstyle:IllegalCatch")
 561     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 562                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 563                                                   String vpnName, String interfaceName) {
 564         if (rd == null) {
 565             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 566                   interfaceName, dpnId, vpnName);
 567             return;
 568         } else {
 569             if (rd.equals(vpnName)) {
 570                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 571                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 572                 return;
 573             }
 574         }
 575         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 576                 interfaceName, dpnId, vpnName, rd);
 577
 578         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 579         if (nextHopIp == null) {
 580             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 581                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 582                     rd, vpnName);
 583             return;
 584         }
 585
 586         try {
 587             //Read NextHops
 588             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 589             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 590                     LogicalDatastoreType.OPERATIONAL, path);
 591             if (adjacencies.isPresent()) {
 592                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 593                 if (!nextHops.isEmpty()) {
 594                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 595                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 596                     VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 597                     long l3vni = vpnInstanceOpData.getL3vni();
 598                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 599                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 600                     for (Adjacency nextHop : nextHops) {
 601                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 602                             continue;
 603                         }
 604                         String gatewayMac = null;
 605                         long label = 0;
 606                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 607                             final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 608                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 609                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 610                                     .get();
 611                         } else {
 612                             label = nextHop.getLabel();
 613                         }
 614                         try {
 615                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 616                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 617                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 618                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 619                                     gatewayMac);
 620                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 621                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 622                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 623                         } catch (Exception e) {
 624                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 625                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 626                                     interfaceName, dpnId, e);
 627                         }
 628                     }
 629                 }
 630             }
 631         } catch (ReadFailedException e) {
 632             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 633                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 634         }
 635     }
 636
 637     // TODO Clean up the exception handling
 638     @SuppressWarnings("checkstyle:IllegalCatch")
 639     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 640                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 641         //Read NextHops
 642         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 643         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 644         if (rd == null) {
 645             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 646                 interfaceName, vpnName);
 647             return;
 648         } else {
 649             if (rd.equals(vpnName)) {
 650                 LOG.info(
 651                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 652                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 653                 return;
 654             }
 655         }
 656         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 657                vpnName, rd);
 658         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 659         try {
 660             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 661                     path);
 662         } catch (ReadFailedException e) {
 663             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 664                     interfaceName, vpnName);
 665         }
 666         if (adjacencies.isPresent()) {
 667             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 668
 669             if (!nextHops.isEmpty()) {
 670                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 671                         nextHops, interfaceName, vpnName, rd);
 672                 for (Adjacency nextHop : nextHops) {
 673                     try {
 674                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 675                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 676                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 677                                     interfaceName, vpnName);
 678                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 679                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 680                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 681                                     interfaceName, vpnName);
 682                         } else {
 683                             // Perform similar operation as interface delete event for extraroutes.
 684                             String allocatedRd = nextHop.getVrfId();
 685                             for (String nh : nextHop.getNextHopIpList()) {
 686                                 deleteExtraRouteFromCurrentAndImportingVpns(
 687                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 688                             }
 689                         }
 690                     } catch (Exception e) {
 691                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 692                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 693                     }
 694                 }
 695             }
 696         }
 697     }
 698
 699     @SuppressWarnings("checkstyle:IllegalCatch")
 700     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 701                                                   String primaryRd, String interfaceName, final long vpnId,
 702                                                   WriteTransaction writeConfigTxn,
 703                                                   WriteTransaction writeOperTxn,
 704                                                   final WriteTransaction writeInvTxn,
 705                                                   Interface interfaceState) {
 706         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 707         // Read NextHops
 708         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 709         Optional<Adjacencies> adjacencies = Optional.absent();
 710         try {
 711             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 712                     path);
 713         } catch (ReadFailedException e) {
 714             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 715                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 716         }
 717         if (!adjacencies.isPresent()) {
 718             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 719                     null/*gwMac*/, writeOperTxn);
 720             return;
 721         }
 722         // Get the rd of the vpn instance
 723         String nextHopIp = null;
 724         try {
 725             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 726         } catch (Exception e) {
 727             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 728                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 729         }
 730         List<String> nhList = new ArrayList<>();
 731         if (nextHopIp != null) {
 732             nhList.add(nextHopIp);
 733             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 734                     interfaceName, dpnId, vpnName, nhList);
 735         }
 736         Optional<String> gwMac = Optional.absent();
 737         String vpnInterfaceSubnetGwMacAddress = null;
 738         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 739         Long l3vni = vpnInstanceOpData.getL3vni();
 740         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 741         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 742         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 743         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 744         List<Adjacency> value = new ArrayList<>();
 745         for (Adjacency nextHop : nextHops) {
 746             String rd = primaryRd;
 747             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 748             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 749                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 750                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 751                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 752                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 753                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 754                 continue;
 755             }
 756             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 757                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 758                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 759                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 760                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 761                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 762                 writeOperTxn.merge(
 763                     LogicalDatastoreType.OPERATIONAL,
 764                     VpnUtil.getPrefixToInterfaceIdentifier(
 765                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 766                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 767                             prefixCue), true);
 768                 final Uuid subnetId = nextHop.getSubnetId();
 769
 770                 String gatewayIp = nextHop.getSubnetGatewayIp();
 771                 if (gatewayIp == null) {
 772                     Optional<String> gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 773                     if (gatewayIpOptional.isPresent()) {
 774                         gatewayIp = gatewayIpOptional.get();
 775                     }
 776                 }
 777
 778                 if (gatewayIp != null) {
 779                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 780                     if (gwMac.isPresent()) {
 781                         // A valid mac-address is available for this subnet-gateway-ip
 782                         // Use this for programming ARP_RESPONDER table here.  And save this
 783                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 784                         // to populate L3_GW_MAC_TABLE there.
 785                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 786                                 gatewayIp, gwMac.get());
 787                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 788                     } else {
 789                         // A valid mac-address is not available for this subnet-gateway-ip
 790                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 791                         // Save this connected-mac-address as gateway-mac-address for the
 792                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 793                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 794                         if (gwMac.isPresent()) {
 795                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 796                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 797                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 798                                     gatewayIp, gwMac.get());
 799                         } else {
 800                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 801                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 802                                 + "gwIp {}",
 803                                 subnetId, interfaceName, vpnName, gatewayIp);
 804                         }
 805                     }
 806                 } else {
 807                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 808                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 809                         subnetId, interfaceName, vpnName);
 810                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 811                 }
 812                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 813                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 814             } else {
 815                 //Extra route adjacency
 816                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 817                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 818                 synchronized (vpnPrefixKey.intern()) {
 819                     java.util.Optional<String> rdToAllocate = VpnUtil
 820                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 821                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 822                     if (rdToAllocate.isPresent()) {
 823                         rd = rdToAllocate.get();
 824                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 825                             rd, prefix);
 826                     } else {
 827                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 828                         continue;
 829                     }
 830                 }
 831                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 832                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 833                         interfaceName, dpnId);
 834             }
 835             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 836             // can be different from the gateway-mac-address within the VRFEntry as the
 837             // gateway-mac-address is a superset.
 838             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 839                     : RouteOrigin.STATIC;
 840             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 841                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 842                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 843             Adjacency operationalAdjacency = null;
 844             try {
 845                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 846             } catch (NullPointerException e) {
 847                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 848                     input, e.getMessage());
 849                 return;
 850             }
 851             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 852                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 853                         vpnName, l3vni, origin,
 854                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 855             }
 856             value.add(operationalAdjacency);
 857         }
 858
 859         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 860         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 861                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 862
 863         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 864                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 865                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 866
 867         for (Adjacency nextHop : aug.getAdjacency()) {
 868             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 869             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 870                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 871                         : RouteOrigin.STATIC;
 872                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 873                 registeredPopulator.populateFib(input, writeConfigTxn);
 874             }
 875         }
 876     }
 877
 878     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 879                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 880         VpnInterfaceOpDataEntry opInterface =
 881               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 882         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 883             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 884         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 885                 WriteTransaction.CREATE_MISSING_PARENTS);
 886         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 887                 interfaceName, dpnId, vpnName);
 888     }
 889
 890     // TODO Clean up the exception handling
 891     @SuppressWarnings("checkstyle:IllegalCatch")
 892     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 893                                            StateTunnelList stateTunnelList,
 894                                            WriteTransaction writeConfigTxn,
 895                                            WriteTransaction writeOperTxn) {
 896
 897         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 898         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 899         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 900         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 901         if (adjList.isEmpty()) {
 902             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 903                     vpnInterface, srcDpnId);
 904             return;
 905         }
 906         String prefix = null;
 907         long label = 0;
 908         List<Adjacency> value = new ArrayList<>();
 909         boolean isNextHopAddReqd = false;
 910         String vpnName = vpnInterface.getVpnInstanceName();
 911         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 912         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 913         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 914                 vpnInterface.getName(), vpnInterface.getDpnId(),
 915                 vpnInterface.getVpnInstanceName(), adjList);
 916         for (Adjacency adj : adjList) {
 917             String rd = adj.getVrfId();
 918             rd = rd != null ? rd : vpnName;
 919             prefix = adj.getIpAddress();
 920             label = adj.getLabel();
 921             List<String> nhList = Collections.singletonList(srcTepIp);
 922             List<String> nextHopList = adj.getNextHopIpList();
 923             // If TEP is added , update the nexthop of primary adjacency.
 924             // Secondary adj nexthop is already pointing to primary adj IP address.
 925             if (nextHopList != null && !nextHopList.isEmpty()) {
 926                 /* everything right already */
 927             } else {
 928                 isNextHopAddReqd = true;
 929             }
 930
 931             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 932                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 933             } else {
 934                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 935                 if (!vrfEntryOptional.isPresent()) {
 936                     continue;
 937                 }
 938                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 939                 if (!nhList.contains(srcTepIp)) {
 940                     nhList.add(srcTepIp);
 941                     isNextHopAddReqd = true;
 942                 }
 943                 value.add(adj);
 944             }
 945
 946             if (isNextHopAddReqd) {
 947                 updateLabelMapper(label, nhList);
 948                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 949                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 950                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 951                 // Update the VRF entry with nextHop
 952                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 953                         label, true, writeConfigTxn);
 954
 955                 //Get the list of VPN's importing this route(prefix) .
 956                 // Then update the VRF entry with nhList
 957                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 958                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 959                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 960                     String vpnRd = vpn.getVrfId();
 961                     if (vpnRd != null) {
 962                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 963                                 srcTepIp, label, true, writeConfigTxn);
 964                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 965                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 966                             vpnInterface.getName(), srcDpnId, vpnName,
 967                             vpn.getVpnInstanceName(), vpnRd);
 968                     }
 969                 }
 970                 // Advertise the prefix to BGP only for external vpn
 971                 // since there is a nexthop change.
 972                 try {
 973                     if (!rd.equalsIgnoreCase(vpnName)) {
 974                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 975                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 976                                 null /*gatewayMacAddress*/);
 977                     }
 978                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 979                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 980                             srcDpnId, vpnName);
 981                 } catch (Exception ex) {
 982                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 983                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 984                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 985                 }
 986             }
 987         }
 988         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 989         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 990                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 991                 .addAugmentation(AdjacenciesOp.class, aug).build();
 992         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 993                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 994         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 995                 WriteTransaction.CREATE_MISSING_PARENTS);
 996         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 997                 vpnInterface.getName(), srcDpnId, vpnName);
 998
 999     }
1000
1001     // TODO Clean up the exception handling
1002     @SuppressWarnings("checkstyle:IllegalCatch")
1003     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1004                                               StateTunnelList stateTunnelList,
1005                                               WriteTransaction writeConfigTxn,
1006                                               WriteTransaction writeOperTxn) {
1007
1008         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1009         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1010         String prefix = null;
1011         long label = 0;
1012         boolean isNextHopRemoveReqd = false;
1013         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1014         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1015         String vpnName = vpnInterface.getVpnInstanceName();
1016         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1017         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1018         if (adjList != null) {
1019             List<Adjacency> value = new ArrayList<>();
1020             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1021                     vpnInterface.getName(), vpnInterface.getDpnId(),
1022                     vpnInterface.getVpnInstanceName(), adjList);
1023             for (Adjacency adj : adjList) {
1024                 List<String> nhList = new ArrayList<>();
1025                 String rd = adj.getVrfId();
1026                 rd = rd != null ? rd : vpnName;
1027                 prefix = adj.getIpAddress();
1028                 List<String> nextHopList = adj.getNextHopIpList();
1029                 label = adj.getLabel();
1030                 if (nextHopList != null && !nextHopList.isEmpty()) {
1031                     isNextHopRemoveReqd = true;
1032                 }
1033                 // If TEP is deleted , remove the nexthop from primary adjacency.
1034                 // Secondary adj nexthop will continue to point to primary adj IP address.
1035                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1036                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1037                 } else {
1038                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1039                     if (!vrfEntryOptional.isPresent()) {
1040                         continue;
1041                     }
1042                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1043                     if (nhList.contains(srcTepIp)) {
1044                         nhList.remove(srcTepIp);
1045                         isNextHopRemoveReqd = true;
1046                     }
1047                     value.add(adj);
1048                 }
1049
1050                 if (isNextHopRemoveReqd) {
1051                     updateLabelMapper(label, nhList);
1052                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1053                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1054                             prefix, nhList, vpnName,
1055                             vpnId, rd, vpnInterface.getName());
1056                     // Update the VRF entry with removed nextHop
1057                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1058                             label, false, writeConfigTxn);
1059
1060                     //Get the list of VPN's importing this route(prefix) .
1061                     // Then update the VRF entry with nhList
1062                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1063                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1064                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1065                         String vpnRd = vpn.getVrfId();
1066                         if (vpnRd != null) {
1067                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1068                                     srcTepIp, label, false, writeConfigTxn);
1069                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1070                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1071                                     nhList, label, vpnInterface.getName(), srcDpnId,
1072                                     vpnName,
1073                                     vpn.getVpnInstanceName(), vpnRd);
1074                         }
1075                     }
1076
1077                     // Withdraw prefix from BGP only for external vpn.
1078                     try {
1079                         if (!rd.equalsIgnoreCase(vpnName)) {
1080                             bgpManager.withdrawPrefix(rd, prefix);
1081                         }
1082                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1083                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1084                                 vpnInterface.getName(), srcDpnId,
1085                                 vpnName);
1086                     } catch (Exception ex) {
1087                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1088                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1089                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1090                     }
1091                 }
1092             }
1093             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1094             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1095                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1096                     .addAugmentation(AdjacenciesOp.class, aug).build();
1097             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1098                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1099             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1100                     WriteTransaction.CREATE_MISSING_PARENTS);
1101             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1102                          vpnInterface.getName(), srcDpnId, vpnName);
1103         }
1104     }
1105
1106     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1107         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1108
1109         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1110         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1111         if (vpnInstanceOpDataEntry == null) {
1112             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1113                     + " to check for vpns exporting the routes", vpnName);
1114             return vpnsToExportRoute;
1115         }
1116
1117         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1118             if (input.getVpnInstanceName() == null) {
1119                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1120                         input.getVrfId());
1121                 return false;
1122             }
1123             return !input.getVpnInstanceName().equals(vpnName);
1124         };
1125
1126         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1127             Iterable<String> commonRTs =
1128                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1129                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1130             return Iterators.size(commonRTs.iterator()) > 0;
1131         };
1132
1133         vpnsToExportRoute =
1134                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1135                         Collectors.toList());
1136         return vpnsToExportRoute;
1137     }
1138
1139     // TODO Clean up the exception handling
1140     @SuppressWarnings("checkstyle:IllegalCatch")
1141     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1142         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1143         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1144             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1145             if (vrfEntries != null) {
1146                 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1147                     for (VrfEntry vrfEntry : vrfEntries) {
1148                         try {
1149                             if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1150                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1151                                 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1152                                                 + " is not a controller managed non intervpn link route. Ignoring.",
1153                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1154                                 continue;
1155                             }
1156                             String prefix = vrfEntry.getDestPrefix();
1157                             String gwMac = vrfEntry.getGatewayMacAddress();
1158                             vrfEntry.getRoutePaths().forEach(routePath -> {
1159                                 String nh = routePath.getNexthopAddress();
1160                                 int label = routePath.getLabel().intValue();
1161                                 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1162                                         vrfEntry.getOrigin()))) {
1163                                     LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1164                                                     + " nexthop {} label {} to vpn {} vpnRd {}",
1165                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1166                                     fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1167                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1168                                             0 /*l3vni*/, gwMac,  vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1169                                             confTx);
1170                                 } else {
1171                                     LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1172                                                     + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1173                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1174                                     SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1175                                     importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1176                                             confTx);
1177                                 }
1178                             });
1179                         } catch (RuntimeException e) {
1180                             LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1181                                             + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1182                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1183                         }
1184                     }
1185                 }), LOG, "Error handing VPN exporting routes");
1186             } else {
1187                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1188                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1189             }
1190         }
1191     }
1192
1193     @Override
1194     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1195         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1196         final String interfaceName = key.getName();
1197         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1198             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1199             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1200         }
1201     }
1202
1203     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1204                                 final VpnInterface vpnInterface, final String vpnName,
1205                                 final String interfaceName) {
1206         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1207             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1208                 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1209                     deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1210                     LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1211                 });
1212                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1213                         vpnInterface.getName(), vpnName);
1214                 return Collections.singletonList(future);
1215             }, DJC_MAX_RETRIES);
1216         } else {
1217             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1218             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1219         }
1220     }
1221
1222     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1223     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1224                                 final VpnInterface vpnInterface, final String vpnName,
1225                                 final String interfaceName, final Interface interfaceState) {
1226         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1227                 vpnName, vpnInterface.getDpnId());
1228         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1229         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1230             () -> {
1231                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1232                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1233                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1234                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1235                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1236                                     interfaceName, vpnName);
1237                             BigInteger dpId = BigInteger.ZERO;
1238                             int ifIndex = 0;
1239                             String gwMacAddress = null;
1240                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1241                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1242                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1243                             try {
1244                                 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1245                                         LogicalDatastoreType.OPERATIONAL, interfaceId);
1246                             } catch (ReadFailedException e) {
1247                                 LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName,
1248                                         vpnName);
1249                                 return;
1250                             }
1251                             if (interfaceState != null) {
1252                                 try {
1253                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1254                                 } catch (NumberFormatException | IllegalStateException e) {
1255                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1256                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1257                                                     + " from vpn interface op data store. ", interfaceName,
1258                                             vpnInterface.getDpnId(), vpnName, e);
1259                                     dpId = BigInteger.ZERO;
1260                                 }
1261                                 ifIndex = interfaceState.getIfIndex();
1262                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1263                             } else {
1264                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1265                                         + " from vpn interface op.", interfaceName);
1266                                 if (optVpnInterface.isPresent()) {
1267                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1268                                     dpId = vpnOpInterface.getDpnId();
1269                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1270                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1271                                 } else {
1272                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1273                                                     + " as interfaceState and vpn interface op is not"
1274                                             + " available", interfaceName, vpnName);
1275                                     return;
1276                                 }
1277                             }
1278                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1279                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1280                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1281                             LOG.info(
1282                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1283                                             + "successfully",
1284                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1285                         })))));
1286                 futures.add(configFuture);
1287                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1288                 return futures;
1289             }, DJC_MAX_RETRIES);
1290     }
1291
1292     protected void processVpnInterfaceDown(BigInteger dpId,
1293                                            String interfaceName,
1294                                            int lportTag,
1295                                            String gwMac,
1296                                            VpnInterfaceOpDataEntry vpnOpInterface,
1297                                            boolean isInterfaceStateDown,
1298                                            WriteTransaction writeConfigTxn,
1299                                            WriteTransaction writeOperTxn,
1300                                            WriteTransaction writeInvTxn) {
1301         if (vpnOpInterface == null) {
1302             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1303                     + " as it is not available in operational data store", interfaceName, dpId);
1304             return;
1305         }
1306         final String vpnName = vpnOpInterface.getVpnInstanceName();
1307         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1308                                                     interfaceName, vpnName);
1309         if (!isInterfaceStateDown) {
1310             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1311             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1312                     null);
1313             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1314             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1315                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1316             if (interfaceManager.isExternalInterface(interfaceName)) {
1317                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1318                         NwConstants.DEL_FLOW);
1319             }
1320             if (!isBgpVpnInternetVpn) {
1321                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1322             }
1323             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1324                     + " successful", interfaceName, dpId, vpnName);
1325         } else {
1326             // Interface is retained in the DPN, but its Link Down.
1327             // Only withdraw the prefixes for this interface from BGP
1328             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1329         }
1330     }
1331
1332     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1333                                           final String vpnName, final long vpnId, String gwMac,
1334                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1335                                           final WriteTransaction writeInvTxn) {
1336         //Read NextHops
1337         try {
1338             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1339                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1340             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1341             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1342                     LogicalDatastoreType.OPERATIONAL, path);
1343             String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1344             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1345                     interfaceName, dpnId, vpnName, primaryRd);
1346             if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1347                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1348                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1349                         interfaceName, dpnId, vpnName, nextHops);
1350                 for (Adjacency nextHop : nextHops) {
1351                     if (nextHop.isPhysNetworkFunc()) {
1352                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1353                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1354                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1355                                 null/*writeCfgTxn*/);
1356                     } else {
1357                         String rd = nextHop.getVrfId();
1358                         List<String> nhList;
1359                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1360                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1361                         } else {
1362                             // This is a primary adjacency
1363                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1364                                     : Collections.emptyList();
1365                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1366                                     interfaceName, writeInvTxn);
1367                         }
1368                         if (!nhList.isEmpty()) {
1369                             if (rd.equals(vpnName)) {
1370                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1371                                 //remove from FIB directly
1372                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1373                                         interfaceName, dpnId, writeConfigTxn));
1374                             } else {
1375                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1376                                         interfaceName, writeConfigTxn);
1377                             }
1378                         } else {
1379                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1380                                             + " interface {}", nextHop.getIpAddress(), rd,
1381                                     nextHop.getAdjacencyType().toString(), interfaceName);
1382                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1383                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1384                         }
1385                     }
1386                     String ip = nextHop.getIpAddress().split("/")[0];
1387                     LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1388                     if (vpnVipToPort != null) {
1389                         VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip, null);
1390                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1391                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1392                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1393                     }
1394                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ip);
1395                     if (vpnPortipToPort != null) {
1396                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1397                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1398                                  + "Interface {} ip {} on dpn {} for vpn {}",
1399                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1400                     }
1401                 }
1402             } else {
1403                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1404                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1405                         + " Removing it.", interfaceName, vpnName, dpnId);
1406                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1407             }
1408         } catch (ReadFailedException e) {
1409             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1410                     interfaceName, dpnId, vpnName);
1411         }
1412     }
1413
1414     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1415                                                                    String interfaceName, BigInteger dpnId,
1416                                                                    WriteTransaction writeConfigTxn) {
1417         return (nh) -> {
1418             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1419                     writeConfigTxn);
1420             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1421                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1422                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1423         };
1424     }
1425
1426     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1427                                            BigInteger dpnId, String rd, String interfaceName,
1428                                            WriteTransaction writeConfigTxn) {
1429         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1430                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1431         nhList.forEach((nh) -> {
1432             //IRT: remove routes from other vpns importing it
1433             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1434                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1435             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1436                 String vpnRd = vpn.getVrfId();
1437                 if (vpnRd != null) {
1438                     fibManager.removeOrUpdateFibEntry(vpnRd,
1439                             nextHop.getIpAddress(), nh, writeConfigTxn);
1440                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1441                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1442                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1443                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1444                 }
1445             }
1446         });
1447     }
1448
1449     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1450                                                  int lportTag, String gwMac, String interfaceName,
1451                                                  WriteTransaction writeInvTxn) {
1452         final Uuid subnetId = nextHop.getSubnetId();
1453         if (nextHop.getSubnetGatewayMacAddress() == null) {
1454             // A valid mac-address was not available for this subnet-gateway-ip
1455             // So a connected-mac-address was used for this subnet and we need
1456             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1457             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1458                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1459         }
1460         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1461                 subnetId);
1462     }
1463
1464     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1465                                                   String interfaceName) {
1466         // This is either an extra-route (or) a learned IP via subnet-route
1467         List<String> nhList = null;
1468         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1469         if (nextHopIp == null || nextHopIp.isEmpty()) {
1470             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1471                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1472                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1473                     vpnName);
1474             nhList = Collections.emptyList();
1475         } else {
1476             nhList = Collections.singletonList(nextHopIp);
1477         }
1478         return nhList;
1479     }
1480
1481     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1482         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1483         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1484         // else use connected interface
1485         if (gwPort != null && gwPort.isSubnetIp()) {
1486             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1487                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1488             return Optional.of(gwPort.getMacAddress());
1489         }
1490         return Optional.absent();
1491     }
1492
1493     @Override
1494     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1495         final VpnInterface update) {
1496         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1497                 update.getDpnId(), original.getVpnInstanceNames(),
1498                 update.getVpnInstanceNames());
1499         final String vpnInterfaceName = update.getName();
1500         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1501         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1502         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1503             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1504         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1505         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1506             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1507
1508         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1509         //handles switching between <internal VPN - external VPN>
1510         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1511             LOG.info("update: handled VPNInterface {} on dpn {} update"
1512                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1513                      original.getName(), dpnId,
1514                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1515                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1516             return;
1517         }
1518         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1519             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1520             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1521             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1522             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1523             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1524                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1525                     // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1526                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1527                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1528                         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(operTx -> {
1529                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1530                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1531                             LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1532                                     update.getName(), newVpnName);
1533                             //handle both addition and removal of adjacencies
1534                             //currently, new adjacency may be an extra route
1535                             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1536                             if (!oldAdjs.equals(newAdjs)) {
1537                                 for (Adjacency adj : copyNewAdjs) {
1538                                     if (copyOldAdjs.contains(adj)) {
1539                                         copyOldAdjs.remove(adj);
1540                                     } else {
1541                                         // add new adjacency - right now only extra route will hit this path
1542                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker,
1543                                                 adj)) {
1544                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1545                                                     dpnId, operTx, confTx);
1546                                         }
1547                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1548                                                 + " vpn interface {} on vpn {} dpnId {}",
1549                                                 adj.getIpAddress(), adj.getNextHopIpList(),
1550                                                 adj.getLabel(), adj.getSubnetId(), update.getName(),
1551                                                 newVpnName, dpnId);
1552                                     }
1553                                 }
1554                                 for (Adjacency adj : copyOldAdjs) {
1555                                     if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker,
1556                                             adj)) {
1557                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1558                                                 && !adj.isPhysNetworkFunc()) {
1559                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1560                                                     operTx, confTx);
1561                                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = operTx.read(
1562                                                     LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier).get();
1563                                             VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get();
1564                                             long vpnId = VpnUtil.getVpnId(dataBroker, newVpnName);
1565                                             VpnUtil.removePrefixToInterfaceAdj(dataBroker, adj, vpnId,
1566                                                     vpnInterfaceOpDataEntry, operTx);
1567                                             //remove FIB entry
1568                                             String vpnRd = VpnUtil.getVpnRd(dataBroker, newVpnName);
1569                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1570                                                     + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1571                                             //remove BGP entry
1572                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1573                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1574                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1575                                             }
1576                                         } else {
1577                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1578                                                     operTx, confTx);
1579                                         }
1580                                     }
1581                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1582                                                     + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1583                                                     .getNextHopIpList(),
1584                                             adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1585                                 }
1586                             }
1587                         }));
1588                     }));
1589                     for (ListenableFuture<Void> future : futures) {
1590                         ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1591                                 update.getName(), update.getVpnInstanceNames());
1592                     }
1593                     return futures;
1594                 });
1595             } else {
1596                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1597                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1598             }
1599         }
1600     }
1601
1602     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1603                                                  VpnInterface original, VpnInterface update) {
1604         boolean isSwap = Boolean.FALSE;
1605         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1606         final String interfaceName = key.getName();
1607         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1608             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1609         List<String> oldVpnListCopy = new ArrayList<>();
1610         oldVpnListCopy.addAll(oldVpnList);
1611         List<String> newVpnList = update.getVpnInstanceNames().stream()
1612             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1613         oldVpnList.removeAll(newVpnList);
1614         newVpnList.removeAll(oldVpnListCopy);
1615         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1616             for (String oldVpnName: oldVpnList) {
1617                 isSwap = Boolean.TRUE;
1618                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1619                         + " running config-driven swap removal", interfaceName, oldVpnName);
1620                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1621                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1622                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1623             }
1624             //Wait for previous interface bindings to be removed
1625             try {
1626                 Thread.sleep(2000);
1627             } catch (InterruptedException e) {
1628                 //Ignore
1629             }
1630             for (String newVpnName: newVpnList) {
1631                 String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1632                 isSwap = Boolean.TRUE;
1633                 if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1634                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1635                             + "running config-driven swap addition", interfaceName, newVpnName);
1636                     final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1637                     final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1638                             ? origAdjs.getAdjacency() : new ArrayList<>();
1639                     final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1640                     final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1641                             ? updateAdjs.getAdjacency() : new ArrayList<>();
1642
1643                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1644                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1645                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1646                             interfaceName, oldVpnListCopy, newVpnName);
1647                 }
1648             }
1649         }
1650         return isSwap;
1651     }
1652
1653     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1654         try {
1655             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1656             synchronized (label.toString().intern()) {
1657                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1658                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1659                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1660                         LogicalDatastoreType.OPERATIONAL, lriIid);
1661                 if (opResult.isPresent()) {
1662                     LabelRouteInfo labelRouteInfo =
1663                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1664                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1665                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1666                 }
1667             }
1668             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1669                     nextHopIpList);
1670         } catch (ReadFailedException e) {
1671             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1672                     nextHopIpList);
1673         } catch (TransactionCommitFailedException e) {
1674             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1675                     nextHopIpList);
1676         }
1677     }
1678
1679     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1680         SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1681
1682         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1683         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1684                 .addAugmentation(SubnetRoute.class, route).build();
1685         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1686         InstanceIdentifierBuilder<VrfTables> idBuilder =
1687             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1688         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1689         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1690         if (writeConfigTxn != null) {
1691             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1692         } else {
1693             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1694         }
1695         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1696                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1697     }
1698
1699     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1700                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1701                                            WriteTransaction writeConfigTxn) {
1702         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1703         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1704         try {
1705             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1706                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1707             if (optVpnInterface.isPresent()) {
1708                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1709                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1710                 String vpnName = currVpnIntf.getVpnInstanceName();
1711                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1712                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1713                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1714                         LogicalDatastoreType.OPERATIONAL, adjPath);
1715                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1716                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1717                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1718                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1719                 List<Adjacency> adjacencies;
1720                 if (optAdjacencies.isPresent()) {
1721                     adjacencies = optAdjacencies.get().getAdjacency();
1722                 } else {
1723                     // This code will be hit in case of first PNF adjacency
1724                     adjacencies = new ArrayList<>();
1725                 }
1726                 long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1727                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1728                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1729                 Adjacency operationalAdjacency = null;
1730                 //Handling dual stack neutron port primary adjacency
1731                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1732                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1733                             currVpnIntf.getName(), vpnName);
1734                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1735                             currVpnIntf.getName());
1736                     if (interfaceState != null) {
1737                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1738                                 currVpnIntf.getName(),
1739                                 vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1740                     }
1741                 }
1742                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1743                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1744                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1745                             : RouteOrigin.STATIC;
1746                     String nh = adj.getNextHopIpList().get(0);
1747                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1748                     synchronized (vpnPrefixKey.intern()) {
1749                         java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1750                                 dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1751                         if (rdToAllocate.isPresent()) {
1752                             input.setRd(rdToAllocate.get());
1753                             operationalAdjacency = populator.createOperationalAdjacency(input);
1754                             int label = operationalAdjacency.getLabel().intValue();
1755                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1756                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1757                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1758                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1759                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1760                                     vpnName, label, currVpnIntf.getName(), dpnId);
1761                         } else {
1762                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1763                                     vpnName, prefix);
1764                             return;
1765                         }
1766                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1767                         // Keeping the MPLS check for now.
1768                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1769                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1770                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1771                                     VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1772                             vpnsToImportRoute.forEach(vpn -> {
1773                                 if (vpn.getVrfId() != null) {
1774                                     VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1775                                             dataBroker, vpn.getVpnId(), vpnId, prefix,
1776                                             VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1777                                             .ifPresent(
1778                                                 rds -> vpnManager.addExtraRoute(
1779                                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1780                                                         adj.getIpAddress(), nh, rds,
1781                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1782                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1783                                                         opAdjacency, encapType, writeConfigTxn));
1784                                 }
1785                             });
1786                         }
1787                     }
1788                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1789                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1790                             currVpnIntf.getName(), vpnName);
1791
1792                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1793
1794                     writeOperTxn.merge(
1795                             LogicalDatastoreType.OPERATIONAL,
1796                             VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1797                                     adj.getSubnetId().getValue()), prefix),
1798                             VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1799                                     adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1800
1801                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1802                             adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1803                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1804
1805                     input.setRd(adj.getVrfId());
1806                 }
1807                 if (operationalAdjacency == null) {
1808                     operationalAdjacency = populator.createOperationalAdjacency(input);
1809                 }
1810                 adjacencies.add(operationalAdjacency);
1811                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1812                 VpnInterfaceOpDataEntry newVpnIntf =
1813                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1814                                 aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1815                                 currVpnIntf.getGatewayMacAddress());
1816
1817                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1818             }
1819         } catch (ReadFailedException e) {
1820             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1821                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1822         }
1823     }
1824
1825     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1826         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1827         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1828     }
1829
1830     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1831             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1832         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1833         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1834         try {
1835             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1836                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1837             if (optVpnInterface.isPresent()) {
1838                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1839                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1840                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1841                         LogicalDatastoreType.OPERATIONAL, path);
1842                 if (optAdjacencies.isPresent()) {
1843                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1844
1845                     if (!adjacencies.isEmpty()) {
1846                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1847                         Iterator<Adjacency> adjIt = adjacencies.iterator();
1848                         while (adjIt.hasNext()) {
1849                             Adjacency adjElem = adjIt.next();
1850                             if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1851                                 String rd = adjElem.getVrfId();
1852                                 adjIt.remove();
1853                                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1854                                 VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1855                                         .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1856                                                 currVpnIntf.getVpnInstanceName(), aug, dpnId,
1857                                                 currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1858                                                 currVpnIntf.getGatewayMacAddress());
1859                                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1860                                 if (adj.getNextHopIpList() != null) {
1861                                     for (String nh : adj.getNextHopIpList()) {
1862                                         deleteExtraRouteFromCurrentAndImportingVpns(
1863                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1864                                                 currVpnIntf.getName(), writeConfigTxn);
1865                                     }
1866                                 } else if (adj.isPhysNetworkFunc()) {
1867                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1868                                             adj.getIpAddress(), adj.getSubnetId());
1869                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1870                                             writeConfigTxn);
1871                                 }
1872                                 break;
1873                             }
1874
1875                         }
1876                     }
1877                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1878                             dpnId, adj.getVrfId());
1879                 } else {
1880                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1881                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1882                 }
1883             }
1884         } catch (ReadFailedException e) {
1885             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1886                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1887         }
1888     }
1889
1890     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1891         String rd, String intfName, WriteTransaction writeConfigTxn) {
1892         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1893         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1894         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1895             String vpnRd = vpn.getVrfId();
1896             if (vpnRd != null) {
1897                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1898             }
1899         }
1900     }
1901
1902     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1903         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1904             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1905             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1906     }
1907
1908     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1909         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1910             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1911     }
1912
1913     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1914                                                     WriteTransaction writeConfigTxn, String vpnName) {
1915         if (vpnInterface == null) {
1916             return;
1917         }
1918         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1919         if (adjs == null) {
1920             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1921                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1922             return;
1923         }
1924         for (Adjacency adj : adjs) {
1925             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1926                 String primaryInterfaceIp = adj.getIpAddress();
1927                 String macAddress = adj.getMacAddress();
1928                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1929
1930                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1931                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1932
1933                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1934                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1935                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1936                         routerInt, label, writeConfigTxn);
1937                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1938                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1939                         macAddress);
1940                 return;
1941             }
1942         }
1943         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1944                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1945                 primaryRd, vpnName);
1946     }
1947
1948     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1949                                                     WriteTransaction writeConfigTxn, String vpnName) {
1950         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
1951         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1952         if (adjs != null) {
1953             List<Adjacency> adjsList = adjs.getAdjacency();
1954             for (Adjacency adj : adjsList) {
1955                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1956                     String primaryInterfaceIp = adj.getIpAddress();
1957                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1958                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1959                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1960                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1961                     return;
1962                 }
1963             }
1964         } else {
1965             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1966                     vpnInterface.getName(), rd);
1967         }
1968     }
1969
1970     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1971         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1972             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1973                   vpnName, intefaceData.vpnInterface.getName());
1974             return;
1975         }
1976         final VpnInterfaceKey key = intefaceData.identifier
1977                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1978         final String interfaceName = key.getName();
1979         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1980                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1981         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1982                   intefaceData.identifier, vpnName);
1983         return;
1984     }
1985
1986     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1987                                               VpnInterface vpnInterface, String vpnName) {
1988         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
1989                .get(vpnName);
1990         if (vpnInterfaces == null) {
1991             vpnInterfaces = new ConcurrentLinkedQueue<>();
1992         }
1993         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
1994         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
1995         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
1996                 vpnInterface.getName(), vpnName);
1997     }
1998
1999     public boolean isVpnInstanceReady(String vpnInstanceName) {
2000         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2001         if (vpnRd == null) {
2002             return false;
2003         }
2004         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2005
2006         return vpnInstanceOpDataEntry != null;
2007     }
2008
2009     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2010         synchronized (vpnInstanceName.intern()) {
2011             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2012                     unprocessedVpnInterfaces.get(vpnInstanceName);
2013             if (vpnInterfaces != null) {
2014                 while (!vpnInterfaces.isEmpty()) {
2015                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2016                     if (hasVpnInstanceCreatedSuccessfully) {
2017                         processSavedInterface(savedInterface, vpnInstanceName);
2018                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2019                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2020                     } else {
2021                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2022                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2023                     }
2024                 }
2025             } else {
2026                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2027             }
2028         }
2029     }
2030
2031     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2032             VpnInterface vpnInterface) {
2033         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2034             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2035                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2036             if (vpnInterfaces != null) {
2037                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2038                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2039                             + "unprocessed list", vpnInterface.getName(),
2040                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2041                 }
2042             } else {
2043                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2044                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2045             }
2046         }
2047     }
2048
2049     public void vpnInstanceIsReady(String vpnInstanceName) {
2050         processSavedInterfaces(vpnInstanceName, true);
2051     }
2052
2053     public void vpnInstanceFailed(String vpnInstanceName) {
2054         processSavedInterfaces(vpnInstanceName, false);
2055     }
2056
2057     private static class UnprocessedVpnInterfaceData {
2058         InstanceIdentifier<VpnInterface> identifier;
2059         VpnInterface vpnInterface;
2060
2061         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2062             this.identifier = identifier;
2063             this.vpnInterface = vpnInterface;
2064         }
2065
2066         @Override
2067         public int hashCode() {
2068             final int prime = 31;
2069             int result = 1;
2070             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2071             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2072             return result;
2073         }
2074
2075         @Override
2076         public boolean equals(Object obj) {
2077             if (this == obj) {
2078                 return true;
2079             }
2080             if (obj == null) {
2081                 return false;
2082             }
2083             if (getClass() != obj.getClass()) {
2084                 return false;
2085             }
2086             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2087             if (identifier == null) {
2088                 if (other.identifier != null) {
2089                     return false;
2090                 }
2091             } else if (!identifier.equals(other.identifier)) {
2092                 return false;
2093             }
2094             if (vpnInterface == null) {
2095                 if (other.vpnInterface != null) {
2096                     return false;
2097                 }
2098             } else if (!vpnInterface.equals(other.vpnInterface)) {
2099                 return false;
2100             }
2101             return true;
2102         }
2103     }
2104
2105     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2106         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2107         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2108         if (vpnInstanceOpData == null) {
2109             return;
2110         }
2111         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2112         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2113             return;
2114         }
2115         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2116         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2117             try {
2118                 InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2119                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2120                 Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker.syncReadOptional(
2121                         dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2122                 if (!vpnInterfaceOptional.isPresent()) {
2123                     return;
2124                 }
2125                 List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2126                         vpnInterface.getInterfaceName());
2127                 if (configVpnAdjacencies == null) {
2128                     LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2129                     return;
2130                 }
2131                 List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2132                         .augmentation(AdjacenciesOp.class).getAdjacency();
2133                 // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2134                 // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2135                 // in operational DS. These unprocessed adjacencies will be handled below.
2136                 // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2137                 // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2138                 configVpnAdjacencies.stream()
2139                         .filter(adjacency -> operationVpnAdjacencies.stream()
2140                                 .noneMatch(operationalAdjacency ->
2141                                         operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2142                         .forEach(adjacency -> {
2143                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2144                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2145                                 () -> {
2146                                     // TODO Deal with sequencing — the config tx must only submitted
2147                                     // if the oper tx goes in
2148                                     if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2149                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2150                                         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
2151                                                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
2152                                                     confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2153                                                             primaryRd, adjacency, vpnInterfaceOptional.get()
2154                                                                     .getDpnId(), confTx, operTx)))));
2155                                         return futures;
2156                                     } else {
2157                                         return Collections.emptyList();
2158                                     }
2159                                 });
2160                         });
2161             } catch (ReadFailedException e) {
2162                 LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2163                         vpnName, primaryRd);
2164             }
2165         }));
2166     }
2167
2168     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2169         private final String interfaceName;
2170         private final boolean add;
2171         private final String txnDestination;
2172
2173         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2174             this.interfaceName = interfaceName;
2175             this.add = add;
2176             this.txnDestination = transactionDest;
2177         }
2178
2179         @Override
2180         public void onSuccess(Void voidObj) {
2181             if (add) {
2182                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2183                         interfaceName, txnDestination);
2184             } else {
2185                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2186             }
2187         }
2188
2189         @Override
2190         public void onFailure(Throwable throwable) {
2191             if (add) {
2192                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2193                         interfaceName, txnDestination, throwable);
2194             } else {
2195                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2196                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
2197             }
2198         }
2199     }
2200 }