vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.netvirt.vpnmanager.VpnUtil.requireNonNullElse;
  15
  16 import com.google.common.base.Optional;
  17 import com.google.common.base.Preconditions;
  18 import com.google.common.collect.Iterators;
  19 import com.google.common.util.concurrent.FutureCallback;
  20 import com.google.common.util.concurrent.Futures;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import com.google.common.util.concurrent.MoreExecutors;
  23 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  24 import java.math.BigInteger;
  25 import java.util.ArrayList;
  26 import java.util.Collections;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ConcurrentHashMap;
  33 import java.util.concurrent.ConcurrentLinkedQueue;
  34 import java.util.concurrent.ExecutionException;
  35 import java.util.function.Consumer;
  36 import java.util.function.Predicate;
  37 import java.util.stream.Collectors;
  38 import javax.annotation.Nullable;
  39 import javax.annotation.PostConstruct;
  40 import javax.annotation.PreDestroy;
  41 import javax.inject.Inject;
  42 import javax.inject.Singleton;
  43 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  44 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  45 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  46 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  47 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  48 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  49 import org.opendaylight.genius.infra.Datastore.Configuration;
  50 import org.opendaylight.genius.infra.Datastore.Operational;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  52 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  53 import org.opendaylight.genius.infra.TransactionAdapter;
  54 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  55 import org.opendaylight.genius.infra.TypedWriteTransaction;
  56 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  57 import org.opendaylight.genius.mdsalutil.NWUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  60 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  61 import org.opendaylight.infrautils.caches.CacheProvider;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  64 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  65 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  66 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  67 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  68 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  69 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  70 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  71 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  72 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  73 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  74 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  78 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  79 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 122 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 123 import org.slf4j.Logger;
 124 import org.slf4j.LoggerFactory;
 125
 126 @Singleton
 127 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 128
 129     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 130     private static final short DJC_MAX_RETRIES = 3;
 131
 132     private final DataBroker dataBroker;
 133     private final ManagedNewTransactionRunner txRunner;
 134     private final IBgpManager bgpManager;
 135     private final IFibManager fibManager;
 136     private final IMdsalApiManager mdsalManager;
 137     private final IdManagerService idManager;
 138     private final OdlInterfaceRpcService ifaceMgrRpcService;
 139     private final VpnFootprintService vpnFootprintService;
 140     private final IInterfaceManager interfaceManager;
 141     private final IVpnManager vpnManager;
 142     private final ArpResponderHandler arpResponderHandler;
 143     private final JobCoordinator jobCoordinator;
 144     private final VpnUtil vpnUtil;
 145
 146     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 147
 148     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 149             new ConcurrentHashMap<>();
 150
 151     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 152
 153     @Inject
 154     public VpnInterfaceManager(final DataBroker dataBroker,
 155                                final IBgpManager bgpManager,
 156                                final IdManagerService idManager,
 157                                final IMdsalApiManager mdsalManager,
 158                                final IFibManager fibManager,
 159                                final OdlInterfaceRpcService ifaceMgrRpcService,
 160                                final VpnFootprintService vpnFootprintService,
 161                                final IInterfaceManager interfaceManager,
 162                                final IVpnManager vpnManager,
 163                                final ArpResponderHandler arpResponderHandler,
 164                                final JobCoordinator jobCoordinator,
 165                                final CacheProvider cacheProvider,
 166                                final VpnUtil vpnUtil) {
 167         super(VpnInterface.class, VpnInterfaceManager.class);
 168
 169         this.dataBroker = dataBroker;
 170         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 171         this.bgpManager = bgpManager;
 172         this.idManager = idManager;
 173         this.mdsalManager = mdsalManager;
 174         this.fibManager = fibManager;
 175         this.ifaceMgrRpcService = ifaceMgrRpcService;
 176         this.vpnFootprintService = vpnFootprintService;
 177         this.interfaceManager = interfaceManager;
 178         this.vpnManager = vpnManager;
 179         this.arpResponderHandler = arpResponderHandler;
 180         this.jobCoordinator = jobCoordinator;
 181         this.vpnUtil = vpnUtil;
 182
 183         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 184                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 185                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 186     }
 187
 188     public Runnable isNotifyTaskQueued(String intfName) {
 189         return vpnIntfMap.remove(intfName);
 190     }
 191
 192     @PostConstruct
 193     public void start() {
 194         LOG.info("{} start", getClass().getSimpleName());
 195         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 196     }
 197
 198     @Override
 199     @PreDestroy
 200     public void close() {
 201         super.close();
 202         vpnInstanceOpDataEntryCache.close();
 203     }
 204
 205     @Override
 206     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 207         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 208     }
 209
 210     @Override
 211     protected VpnInterfaceManager getDataTreeChangeListener() {
 212         return VpnInterfaceManager.this;
 213     }
 214
 215     @Override
 216     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 217         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 218         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 219                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 220         addVpnInterface(identifier, vpnInterface, null, null);
 221     }
 222
 223     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 224                           final VpnInterface vpnInterface, String vpnName) {
 225         synchronized (vpnName.intern()) {
 226             if (isVpnInstanceReady(vpnName)) {
 227                 return true;
 228             }
 229             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 230             return false;
 231         }
 232     }
 233
 234     // TODO Clean up the exception handling
 235     @SuppressWarnings("checkstyle:IllegalCatch")
 236     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 237                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 238         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(vpnInterface.getVpnInstanceNames(),
 239                 Collections.<VpnInstanceNames>emptyList())) {
 240             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 241             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 242         }
 243     }
 244
 245     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 246                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 247         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 248         final String interfaceName = key.getName();
 249
 250         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 251             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 252                   vpnName, vpnInterface.getName());
 253             return;
 254         }
 255         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 256              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 257         List<Adjacency> copyOldAdjs = null;
 258         if (oldAdjs != null) {
 259             copyOldAdjs = new ArrayList<>();
 260             copyOldAdjs.addAll(oldAdjs);
 261         }
 262         List<Adjacency> copyNewAdjs = null;
 263         if (newAdjs != null) {
 264             copyNewAdjs = new ArrayList<>();
 265             copyNewAdjs.addAll(newAdjs);
 266         }
 267         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 268     }
 269
 270     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 271                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 272                     final @Nullable List<Adjacency> newAdjs,
 273                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 274         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 275         final String interfaceName = key.getName();
 276         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 277         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 278             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 279             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 280             if (interfaceState != null) {
 281                 try {
 282                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 283                     final int ifIndex = interfaceState.getIfIndex();
 284                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 285                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 286                         // (the inventory tx goes in last)
 287                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 288                         //set of prefix used, as entry in prefix-to-interface datastore
 289                         // is prerequisite for refresh Fib to avoid race condition leading to
 290                         // missing remote next hop in bucket actions on bgp-vpn delete
 291                         Set<String> prefixListForRefreshFib = new HashSet<>();
 292                         ListenableFuture<Void> confFuture =
 293                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 294                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 295                                     operTx -> futures.add(
 296                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 297                                             LOG.info(
 298                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 299                                                     + " on dpn {}",
 300                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 301                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 302                                                 confTx, operTx, invTx, interfaceState, vpnName,
 303                                                 prefixListForRefreshFib);
 304                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 305                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 306                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 307                                                 if (newAdjs != null) {
 308                                                     for (Adjacency adj : newAdjs) {
 309                                                         if (oldAdjs.contains(adj)) {
 310                                                             oldAdjs.remove(adj);
 311                                                         } else {
 312                                                             if (!isBgpVpnInternetVpn
 313                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 314                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 315                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 316                                                                     prefixListForRefreshFib);
 317                                                             }
 318                                                         }
 319                                                     }
 320                                                 }
 321                                                 for (Adjacency adj : oldAdjs) {
 322                                                     if (!isBgpVpnInternetVpn
 323                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 324                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 325                                                             operTx, confTx);
 326                                                     }
 327                                                 }
 328                                             }
 329                                         })))));
 330                         Futures.addCallback(confFuture,
 331                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 332                             MoreExecutors.directExecutor());
 333                         futures.add(confFuture);
 334                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 335                             MoreExecutors.directExecutor());
 336                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 337                             + " processed successfully", interfaceName, vpnName, dpnId);
 338                         return futures;
 339                     });
 340                 } catch (NumberFormatException | IllegalStateException e) {
 341                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 342                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 343                         vpnName, e);
 344                     return;
 345                 }
 346             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 347                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 348                     () -> {
 349                         ListenableFuture<Void> future =
 350                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 351                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 352                                     confTx, vpnName);
 353                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 354                                     vpnName, vpnInterface.getDpnId());
 355                             });
 356                         ListenableFutures.addErrorLogging(future, LOG,
 357                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 358                         return Collections.singletonList(future);
 359                     });
 360             } else {
 361                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 362                     + " is not available", interfaceName, vpnName);
 363             }
 364         } else {
 365             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 366                     + " as vpn is pending delete", interfaceName, vpnName,
 367                 vpnInterface.getDpnId());
 368         }
 369     }
 370
 371     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 372     // see comments below.
 373     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 374     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 375             final int lportTag, boolean isInterfaceUp,
 376             TypedWriteTransaction<Configuration> writeConfigTxn,
 377             TypedWriteTransaction<Operational> writeOperTxn,
 378             TypedReadWriteTransaction<Configuration> writeInvTxn,
 379             Interface interfaceState, final String vpnName,
 380             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 381         final String interfaceName = vpnInterface.getName();
 382         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 383                 vpnName);
 384         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 385         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 386         if (!isInterfaceUp) {
 387             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 388                      interfaceName, dpId, vpnName);
 389             long vpnId = vpnUtil.getVpnId(vpnName);
 390             if (vpnId == VpnConstants.INVALID_ID) {
 391                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 392                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 393                         dpId);
 394                 return;
 395             }
 396
 397             boolean waitForVpnInterfaceOpRemoval = false;
 398             if (opVpnInterface != null) {
 399                 String opVpnName = opVpnInterface.getVpnInstanceName();
 400                 String primaryInterfaceIp = null;
 401                 if (Objects.equals(opVpnName, vpnName)) {
 402                     // Please check if the primary VRF Entry does not exist for VPNInterface
 403                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 404                     // back to back
 405                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 406                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 407                     if (adjs == null) {
 408                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 409                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 410                                 vpnName);
 411                         return;
 412                     }
 413                     for (Adjacency adj : adjs) {
 414                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 415                             primaryInterfaceIp = adj.getIpAddress();
 416                             break;
 417                         }
 418                     }
 419                     if (primaryInterfaceIp == null) {
 420                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 421                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 422                                 dpId, vpnName);
 423                         return;
 424                     }
 425                     // Get the rd of the vpn instance
 426                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 427                     if (vrf != null) {
 428                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 429                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 430                         return;
 431                     }
 432                     waitForVpnInterfaceOpRemoval = true;
 433                 } else {
 434                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 435                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 436                 }
 437             }
 438             if (!waitForVpnInterfaceOpRemoval) {
 439                 // Add the VPNInterface and quit
 440                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 441                         null/*ipAddressSourceValuePair*/,
 442                         true /* add */);
 443                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 444                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 445                 if (!isBgpVpnInternetVpn) {
 446                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 447                 }
 448                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 449                         dpId, vpnName);
 450                 if (interfaceManager.isExternalInterface(interfaceName)) {
 451                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 452                         NwConstants.ADD_FLOW);
 453                 }
 454                 return;
 455             }
 456
 457             // FIB didn't get a chance yet to clean up this VPNInterface
 458             // Let us give it a chance here !
 459             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 460                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 461             try {
 462                 Runnable notifyTask = new VpnNotifyTask();
 463                 synchronized (notifyTask) {
 464                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 465                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 466                     // published here while holding the lock on it so this path will hit the wait before notify can be
 467                     // invoked.
 468                     vpnIntfMap.put(interfaceName, notifyTask);
 469                     try {
 470                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 471                     } catch (InterruptedException e) {
 472                         // Ignored
 473                     }
 474                 }
 475             } finally {
 476                 vpnIntfMap.remove(interfaceName);
 477             }
 478
 479             if (opVpnInterface != null) {
 480                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 481                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 482                         dpId, vpnName);
 483                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 484                 return;
 485             }
 486             // VPNInterface got removed, proceed with Add
 487             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 488                     interfaceName, dpId, vpnName);
 489             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 490                     null/*ipAddressSourceValuePair*/,
 491                     true /* add */);
 492             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 493                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 494             if (!isBgpVpnInternetVpn) {
 495                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 496             }
 497             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 498                     + " FIB to clean up", interfaceName, dpId, vpnName);
 499             if (interfaceManager.isExternalInterface(interfaceName)) {
 500                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 501                                lportTag, NwConstants.ADD_FLOW);
 502             }
 503         } else {
 504             try {
 505                 // Interface is retained in the DPN, but its Link Up.
 506                 // Advertise prefixes again for this interface to BGP
 507                 InstanceIdentifier<VpnInterface> identifier =
 508                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 509                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 510                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 511                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 512                 // Perform similar operation as interface add event for extraroutes.
 513                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 514                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 515                         LogicalDatastoreType.CONFIGURATION, path);
 516                 if (!optAdjacencies.isPresent()) {
 517                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 518                     return;
 519                 }
 520                 List<Adjacency> adjacencies = requireNonNullElse(optAdjacencies.get().getAdjacency(), emptyList());
 521                 for (Adjacency adjacency : adjacencies) {
 522                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 523                         continue;
 524                     }
 525                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 526                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 527                         continue;
 528                     }
 529                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 530                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 531                 }
 532             } catch (ReadFailedException e) {
 533                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 534                         interfaceName, vpnName, primaryRd, dpId);
 535             }
 536         }
 537     }
 538
 539     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 540         int lportTag, int addOrRemove) {
 541         Uuid extNetworkId;
 542         try {
 543             // vpn instance of ext-net interface is the network-id
 544             extNetworkId = new Uuid(vpnName);
 545         } catch (IllegalArgumentException e) {
 546             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 547                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 548             return;
 549         }
 550
 551         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 552         if (routerIds == null || routerIds.isEmpty()) {
 553             LOG.info("processExternalVpnInterface: No router is associated with {}."
 554                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 555                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 556             return;
 557         }
 558
 559         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 560                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 561         for (Uuid routerId : routerIds) {
 562             String routerName = routerId.getValue();
 563             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 564             if (Objects.equals(primarySwitch, dpId)) {
 565                 Routers router = vpnUtil.getExternalRouter(routerName);
 566                 if (router != null) {
 567                     if (addOrRemove == NwConstants.ADD_FLOW) {
 568                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 569                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 570                                 dpId, interfaceName, lportTag);
 571                     } else {
 572                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 573                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 574                                 dpId, interfaceName, lportTag);
 575                     }
 576                 } else {
 577                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 578                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 579                             interfaceName, dpId, vpnName);
 580                 }
 581             }
 582         }
 583     }
 584
 585     // TODO Clean up the exception handling
 586     @SuppressWarnings("checkstyle:IllegalCatch")
 587     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 588                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 589                                                   String vpnName, String interfaceName) {
 590         if (rd == null) {
 591             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 592                   interfaceName, dpnId, vpnName);
 593             return;
 594         } else {
 595             if (rd.equals(vpnName)) {
 596                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 597                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 598                 return;
 599             }
 600         }
 601         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 602                 interfaceName, dpnId, vpnName, rd);
 603
 604         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 605         if (nextHopIp == null) {
 606             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 607                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 608                     rd, vpnName);
 609             return;
 610         }
 611
 612         try {
 613             //Read NextHops
 614             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 615             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 616                     LogicalDatastoreType.OPERATIONAL, path);
 617             if (adjacencies.isPresent()) {
 618                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 619                 if (nextHops != null && !nextHops.isEmpty()) {
 620                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 621                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 622                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 623                     long l3vni = vpnInstanceOpData.getL3vni();
 624                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 625                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 626                     for (Adjacency nextHop : nextHops) {
 627                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 628                             continue;
 629                         }
 630                         String gatewayMac = null;
 631                         long label = 0;
 632                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 633                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 634                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 635                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 636                                     .get();
 637                         } else {
 638                             label = nextHop.getLabel();
 639                         }
 640                         try {
 641                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 642                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 643                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 644                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 645                                     gatewayMac);
 646                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 647                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 648                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 649                         } catch (Exception e) {
 650                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 651                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 652                                     interfaceName, dpnId, e);
 653                         }
 654                     }
 655                 }
 656             }
 657         } catch (ReadFailedException e) {
 658             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 659                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 660         }
 661     }
 662
 663     // TODO Clean up the exception handling
 664     @SuppressWarnings("checkstyle:IllegalCatch")
 665     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 666                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 667                         TypedWriteTransaction<Operational> writeOperTx) {
 668         //Read NextHops
 669         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 670         String rd = vpnUtil.getVpnRd(interfaceName);
 671         if (rd == null) {
 672             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 673                 interfaceName, vpnName);
 674             return;
 675         } else {
 676             if (rd.equals(vpnName)) {
 677                 LOG.info(
 678                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 679                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 680                 return;
 681             }
 682         }
 683         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 684                vpnName, rd);
 685         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 686         try {
 687             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 688                     path);
 689         } catch (ReadFailedException e) {
 690             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 691                     interfaceName, vpnName);
 692         }
 693         if (adjacencies.isPresent()) {
 694             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 695
 696             if (nextHops != null && !nextHops.isEmpty()) {
 697                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 698                         nextHops, interfaceName, vpnName, rd);
 699                 for (Adjacency nextHop : nextHops) {
 700                     try {
 701                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 702                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 703                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 704                                     interfaceName, vpnName);
 705                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 706                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 707                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 708                                     interfaceName, vpnName);
 709                         } else {
 710                             // Perform similar operation as interface delete event for extraroutes.
 711                             String allocatedRd = nextHop.getVrfId();
 712                             for (String nh : requireNonNullElse(nextHop.getNextHopIpList(),
 713                                     Collections.<String>emptyList())) {
 714                                 deleteExtraRouteFromCurrentAndImportingVpns(
 715                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 716                                     writeOperTx);
 717                             }
 718                         }
 719                     } catch (Exception e) {
 720                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 721                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 722                     }
 723                 }
 724             }
 725         }
 726     }
 727
 728     @SuppressWarnings("checkstyle:IllegalCatch")
 729     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 730                                                   String primaryRd, String interfaceName, final long vpnId,
 731                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 732                                                   TypedWriteTransaction<Operational> writeOperTxn,
 733                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 734                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 735             throws ExecutionException, InterruptedException {
 736         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 737         // Read NextHops
 738         Optional<VpnInterface> vpnInteface = Optional.absent();
 739         try {
 740             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 741             LogicalDatastoreType.CONFIGURATION, identifier);
 742         } catch (ReadFailedException e) {
 743             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 744                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 745         }
 746         Uuid intfnetworkUuid = null;
 747         NetworkType networkType = null;
 748         Long segmentationId = Long.valueOf(-1);
 749         Adjacencies adjacencies = null;
 750         if (vpnInteface.isPresent()) {
 751             intfnetworkUuid = vpnInteface.get().getNetworkId();
 752             networkType = vpnInteface.get().getNetworkType();
 753             segmentationId = vpnInteface.get().getSegmentationId();
 754             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 755             if (adjacencies == null) {
 756                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 757                         null/*gwMac*/, writeOperTxn);
 758                 return;
 759             }
 760         }
 761         // Get the rd of the vpn instance
 762         String nextHopIp = null;
 763         try {
 764             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 765         } catch (Exception e) {
 766             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 767                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 768         }
 769         List<String> nhList = new ArrayList<>();
 770         if (nextHopIp != null) {
 771             nhList.add(nextHopIp);
 772             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 773                     interfaceName, dpnId, vpnName, nhList);
 774         }
 775         Optional<String> gwMac = Optional.absent();
 776         String vpnInterfaceSubnetGwMacAddress = null;
 777         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 778         Long l3vni = vpnInstanceOpData.getL3vni();
 779         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 780         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 781         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 782         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : emptyList();
 783         List<Adjacency> value = new ArrayList<>();
 784         for (Adjacency nextHop : nextHops) {
 785             String rd = primaryRd;
 786             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 787             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 788                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 789                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 790                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 791                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 792                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 793                 continue;
 794             }
 795             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 796                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 797                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 798                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 799                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 800                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 801
 802                 Prefixes prefixes = (intfnetworkUuid != null)
 803                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 804                             segmentationId, prefixCue) :
 805                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 806                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 807                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 808                 final Uuid subnetId = nextHop.getSubnetId();
 809
 810                 String gatewayIp = nextHop.getSubnetGatewayIp();
 811                 if (gatewayIp == null) {
 812                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 813                     if (gatewayIpOptional.isPresent()) {
 814                         gatewayIp = gatewayIpOptional.get();
 815                     }
 816                 }
 817
 818                 if (gatewayIp != null) {
 819                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 820                     if (gwMac.isPresent()) {
 821                         // A valid mac-address is available for this subnet-gateway-ip
 822                         // Use this for programming ARP_RESPONDER table here.  And save this
 823                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 824                         // to populate L3_GW_MAC_TABLE there.
 825                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 826                                 gatewayIp, gwMac.get());
 827                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 828                     } else {
 829                         // A valid mac-address is not available for this subnet-gateway-ip
 830                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 831                         // Save this connected-mac-address as gateway-mac-address for the
 832                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 833                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 834                         if (gwMac.isPresent()) {
 835                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 836                                     NwConstants.ADD_FLOW, gwMac.get());
 837                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 838                                     gatewayIp, gwMac.get());
 839                         } else {
 840                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 841                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 842                                 + "gwIp {}",
 843                                 subnetId, interfaceName, vpnName, gatewayIp);
 844                         }
 845                     }
 846                 } else {
 847                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 848                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 849                         subnetId, interfaceName, vpnName);
 850                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 851                 }
 852                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 853                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 854             } else {
 855                 //Extra route adjacency
 856                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 857                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 858                 synchronized (vpnPrefixKey.intern()) {
 859                     java.util.Optional<String> rdToAllocate = vpnUtil
 860                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 861                                     nextHop.getNextHopIpList().get(0), dpnId);
 862                     if (rdToAllocate.isPresent()) {
 863                         rd = rdToAllocate.get();
 864                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 865                             rd, prefix);
 866                     } else {
 867                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 868                         continue;
 869                     }
 870                 }
 871                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 872                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 873                         interfaceName, dpnId);
 874             }
 875             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 876             // can be different from the gateway-mac-address within the VRFEntry as the
 877             // gateway-mac-address is a superset.
 878             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 879                     : RouteOrigin.STATIC;
 880             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 881                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 882                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 883             Adjacency operationalAdjacency = null;
 884             try {
 885                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 886             } catch (NullPointerException e) {
 887                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 888                     input, e.getMessage());
 889                 return;
 890             }
 891             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 892                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 893                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 894                     writeConfigTxn);
 895             }
 896             value.add(operationalAdjacency);
 897         }
 898
 899         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 900         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 901                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 902
 903         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 904                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 905                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 906
 907         for (Adjacency nextHop : aug.getAdjacency()) {
 908             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 909             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 910                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 911                         : RouteOrigin.STATIC;
 912                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 913                 registeredPopulator.populateFib(input, writeConfigTxn);
 914             }
 915         }
 916     }
 917
 918     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 919                                               long lportTag, String gwMac,
 920                                               TypedWriteTransaction<Operational> writeOperTxn) {
 921         VpnInterfaceOpDataEntry opInterface =
 922               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 923         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 924             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 925         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 926         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 927                 interfaceName, dpnId, vpnName);
 928     }
 929
 930     // TODO Clean up the exception handling
 931     @SuppressWarnings("checkstyle:IllegalCatch")
 932     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 933                                            StateTunnelList stateTunnelList,
 934                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 935                                            TypedWriteTransaction<Operational> writeOperTxn) {
 936
 937         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 938         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 939         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 940         List<Adjacency> adjList =
 941             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 942         if (adjList.isEmpty()) {
 943             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 944                     vpnInterface, srcDpnId);
 945             return;
 946         }
 947         String prefix = null;
 948         long label = 0;
 949         List<Adjacency> value = new ArrayList<>();
 950         boolean isNextHopAddReqd = false;
 951         String vpnName = vpnInterface.getVpnInstanceName();
 952         long vpnId = vpnUtil.getVpnId(vpnName);
 953         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 954         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 955                 vpnInterface.getName(), vpnInterface.getDpnId(),
 956                 vpnInterface.getVpnInstanceName(), adjList);
 957         for (Adjacency adj : adjList) {
 958             String rd = adj.getVrfId();
 959             rd = rd != null ? rd : vpnName;
 960             prefix = adj.getIpAddress();
 961             label = adj.getLabel();
 962             List<String> nhList = Collections.singletonList(srcTepIp);
 963             List<String> nextHopList = adj.getNextHopIpList();
 964             // If TEP is added , update the nexthop of primary adjacency.
 965             // Secondary adj nexthop is already pointing to primary adj IP address.
 966             if (nextHopList == null || nextHopList.isEmpty()) {
 967                 isNextHopAddReqd = true;
 968             }
 969
 970             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 971                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 972             } else {
 973                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 974                 if (!vrfEntryOptional.isPresent()) {
 975                     continue;
 976                 }
 977                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 978                 if (!nhList.contains(srcTepIp)) {
 979                     nhList.add(srcTepIp);
 980                     isNextHopAddReqd = true;
 981                 }
 982                 value.add(adj);
 983             }
 984
 985             if (isNextHopAddReqd) {
 986                 updateLabelMapper(label, nhList);
 987                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 988                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 989                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 990                 // Update the VRF entry with nextHop
 991                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 992                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 993
 994                 //Get the list of VPN's importing this route(prefix) .
 995                 // Then update the VRF entry with nhList
 996                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 997                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 998                     String vpnRd = vpn.getVrfId();
 999                     if (vpnRd != null) {
1000                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1001                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1002                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1003                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1004                             vpnInterface.getName(), srcDpnId, vpnName,
1005                             vpn.getVpnInstanceName(), vpnRd);
1006                     }
1007                 }
1008                 // Advertise the prefix to BGP only for external vpn
1009                 // since there is a nexthop change.
1010                 try {
1011                     if (!rd.equalsIgnoreCase(vpnName)) {
1012                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1013                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
1014                                 null /*gatewayMacAddress*/);
1015                     }
1016                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1017                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1018                             srcDpnId, vpnName);
1019                 } catch (Exception ex) {
1020                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1021                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1022                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1023                 }
1024             }
1025         }
1026         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1027         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1028                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1029                 .addAugmentation(AdjacenciesOp.class, aug).build();
1030         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1031                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1032         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1033         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1034                 vpnInterface.getName(), srcDpnId, vpnName);
1035
1036     }
1037
1038     // TODO Clean up the exception handling
1039     @SuppressWarnings("checkstyle:IllegalCatch")
1040     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1041                                               StateTunnelList stateTunnelList,
1042                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1043                                               TypedWriteTransaction<Operational> writeOperTxn) {
1044
1045         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1046         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1047         String prefix = null;
1048         long label = 0;
1049         boolean isNextHopRemoveReqd = false;
1050         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1051         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1052         String vpnName = vpnInterface.getVpnInstanceName();
1053         long vpnId = vpnUtil.getVpnId(vpnName);
1054         String primaryRd = vpnUtil.getVpnRd(vpnName);
1055         if (adjList != null) {
1056             List<Adjacency> value = new ArrayList<>();
1057             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1058                     vpnInterface.getName(), vpnInterface.getDpnId(),
1059                     vpnInterface.getVpnInstanceName(), adjList);
1060             for (Adjacency adj : adjList) {
1061                 List<String> nhList = new ArrayList<>();
1062                 String rd = adj.getVrfId();
1063                 rd = rd != null ? rd : vpnName;
1064                 prefix = adj.getIpAddress();
1065                 List<String> nextHopList = adj.getNextHopIpList();
1066                 label = adj.getLabel();
1067                 if (nextHopList != null && !nextHopList.isEmpty()) {
1068                     isNextHopRemoveReqd = true;
1069                 }
1070                 // If TEP is deleted , remove the nexthop from primary adjacency.
1071                 // Secondary adj nexthop will continue to point to primary adj IP address.
1072                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1073                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1074                 } else {
1075                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1076                     if (!vrfEntryOptional.isPresent()) {
1077                         continue;
1078                     }
1079                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1080                     if (nhList.contains(srcTepIp)) {
1081                         nhList.remove(srcTepIp);
1082                         isNextHopRemoveReqd = true;
1083                     }
1084                     value.add(adj);
1085                 }
1086
1087                 if (isNextHopRemoveReqd) {
1088                     updateLabelMapper(label, nhList);
1089                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1090                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1091                             prefix, nhList, vpnName,
1092                             vpnId, rd, vpnInterface.getName());
1093                     // Update the VRF entry with removed nextHop
1094                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1095                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1096
1097                     //Get the list of VPN's importing this route(prefix) .
1098                     // Then update the VRF entry with nhList
1099                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1100                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1101                         String vpnRd = vpn.getVrfId();
1102                         if (vpnRd != null) {
1103                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1104                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1105                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1106                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1107                                     nhList, label, vpnInterface.getName(), srcDpnId,
1108                                     vpnName,
1109                                     vpn.getVpnInstanceName(), vpnRd);
1110                         }
1111                     }
1112
1113                     // Withdraw prefix from BGP only for external vpn.
1114                     try {
1115                         if (!rd.equalsIgnoreCase(vpnName)) {
1116                             bgpManager.withdrawPrefix(rd, prefix);
1117                         }
1118                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1119                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1120                                 vpnInterface.getName(), srcDpnId,
1121                                 vpnName);
1122                     } catch (Exception ex) {
1123                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1124                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1125                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1126                     }
1127                 }
1128             }
1129             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1130             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1131                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1132                     .addAugmentation(AdjacenciesOp.class, aug).build();
1133             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1134                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1135             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1136             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1137                          vpnInterface.getName(), srcDpnId, vpnName);
1138         }
1139     }
1140
1141     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1142         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1143
1144         String vpnRd = vpnUtil.getVpnRd(vpnName);
1145         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1146         if (vpnInstanceOpDataEntry == null) {
1147             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1148                     + " to check for vpns exporting the routes", vpnName);
1149             return vpnsToExportRoute;
1150         }
1151
1152         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1153             if (input.getVpnInstanceName() == null) {
1154                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1155                         input.getVrfId());
1156                 return false;
1157             }
1158             return !input.getVpnInstanceName().equals(vpnName);
1159         };
1160
1161         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1162             Iterable<String> commonRTs =
1163                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1164                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1165             return Iterators.size(commonRTs.iterator()) > 0;
1166         };
1167
1168         vpnsToExportRoute =
1169                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1170                         Collectors.toList());
1171         return vpnsToExportRoute;
1172     }
1173
1174     // TODO Clean up the exception handling
1175     @SuppressWarnings("checkstyle:IllegalCatch")
1176     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1177         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1178         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1179             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1180             if (vrfEntries != null) {
1181                 ListenableFutures.addErrorLogging(
1182                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1183                         for (VrfEntry vrfEntry : vrfEntries) {
1184                             try {
1185                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1186                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1187                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1188                                             + " is not a controller managed non intervpn link route. Ignoring.",
1189                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1190                                     continue;
1191                                 }
1192                                 String prefix = vrfEntry.getDestPrefix();
1193                                 String gwMac = vrfEntry.getGatewayMacAddress();
1194                                 requireNonNullElse(vrfEntry.getRoutePaths(),
1195                                     Collections.<RoutePaths>emptyList()).forEach(routePath -> {
1196                                         String nh = routePath.getNexthopAddress();
1197                                         int label = routePath.getLabel().intValue();
1198                                         if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1199                                             vrfEntry.getOrigin()))) {
1200                                             LOG.info(
1201                                                 "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1202                                                     + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1203                                                 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1204                                             fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1205                                                 Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1206                                                 0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1207                                                 confTx);
1208                                         } else {
1209                                             LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1210                                                     + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1211                                                 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1212                                             SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1213                                             importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1214                                                 confTx);
1215                                         }
1216                                     });
1217                             } catch (RuntimeException e) {
1218                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1219                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1220                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1221                             }
1222                         }
1223                     }), LOG, "Error handing VPN exporting routes");
1224             } else {
1225                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1226                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1227             }
1228         }
1229     }
1230
1231     @Override
1232     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1233         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1234         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1235         final String interfaceName = key.getName();
1236         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(vpnInterface.getVpnInstanceNames(),
1237                 Collections.<VpnInstanceNames>emptyList())) {
1238             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1239             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1240         }
1241     }
1242
1243     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1244                                 final VpnInterface vpnInterface, final String vpnName,
1245                                 final String interfaceName) {
1246         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1247             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1248                 ListenableFuture<Void> future =
1249                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1250                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1251                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1252                     });
1253                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1254                         vpnInterface.getName(), vpnName);
1255                 return Collections.singletonList(future);
1256             }, DJC_MAX_RETRIES);
1257         } else {
1258             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1259             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1260         }
1261     }
1262
1263     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1264     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1265                                            final VpnInterface vpnInterface, final String vpnName,
1266                                            final String interfaceName, final Interface interfaceState) {
1267         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1268                 vpnName, vpnInterface.getDpnId());
1269         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1270         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1271             () -> {
1272                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1273                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1274                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1275                         writeOperTxn -> futures.add(
1276                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1277                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1278                                         interfaceName, vpnName);
1279                                 BigInteger dpId;
1280                                 int ifIndex;
1281                                 String gwMacAddress;
1282                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1283                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1284                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1285                                 try {
1286                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1287                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1288                                 } catch (ReadFailedException e) {
1289                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1290                                             interfaceName, vpnName);
1291                                     return;
1292                                 }
1293                                 if (interfaceState != null) {
1294                                     try {
1295                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1296                                     } catch (NumberFormatException | IllegalStateException e) {
1297                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1298                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1299                                                         + " from vpn interface op data store. ", interfaceName,
1300                                                 vpnInterface.getDpnId(), vpnName, e);
1301                                         dpId = BigInteger.ZERO;
1302                                     }
1303                                     ifIndex = interfaceState.getIfIndex();
1304                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1305                                 } else {
1306                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1307                                             + " from vpn interface op.", interfaceName);
1308                                     if (optVpnInterface.isPresent()) {
1309                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1310                                         dpId = vpnOpInterface.getDpnId();
1311                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1312                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1313                                     } else {
1314                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1315                                                 + " as interfaceState and vpn interface op is not"
1316                                                 + " available", interfaceName, vpnName);
1317                                         return;
1318                                     }
1319                                 }
1320                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1321                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1322                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1323                                 LOG.info(
1324                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1325                                                 + "successfully",
1326                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1327                             })))));
1328                 futures.add(configFuture);
1329                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1330                 return futures;
1331             }, DJC_MAX_RETRIES);
1332     }
1333
1334     protected void processVpnInterfaceDown(BigInteger dpId,
1335                                            String interfaceName,
1336                                            int lportTag,
1337                                            String gwMac,
1338                                            VpnInterfaceOpDataEntry vpnOpInterface,
1339                                            boolean isInterfaceStateDown,
1340                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1341                                            TypedWriteTransaction<Operational> writeOperTxn,
1342                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1343             throws ExecutionException, InterruptedException {
1344         if (vpnOpInterface == null) {
1345             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1346                     + " as it is not available in operational data store", interfaceName, dpId);
1347             return;
1348         }
1349         final String vpnName = vpnOpInterface.getVpnInstanceName();
1350         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1351                                                     interfaceName, vpnName);
1352         if (!isInterfaceStateDown) {
1353             final long vpnId = vpnUtil.getVpnId(vpnName);
1354             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1355             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1356             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1357                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1358             if (interfaceManager.isExternalInterface(interfaceName)) {
1359                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1360                     NwConstants.DEL_FLOW);
1361             }
1362             if (!isBgpVpnInternetVpn) {
1363                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1364             }
1365             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1366                     + " successful", interfaceName, dpId, vpnName);
1367         } else {
1368             // Interface is retained in the DPN, but its Link Down.
1369             // Only withdraw the prefixes for this interface from BGP
1370             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1371         }
1372     }
1373
1374     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1375                                           final String vpnName, final long vpnId, String gwMac,
1376                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1377                                           TypedWriteTransaction<Operational> writeOperTxn,
1378                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1379             throws ExecutionException, InterruptedException {
1380         //Read NextHops
1381         try {
1382             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1383                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1384             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1385             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1386                     LogicalDatastoreType.OPERATIONAL, path);
1387             boolean isNonPrimaryAdjIp = Boolean.FALSE;
1388             String primaryRd = vpnUtil.getVpnRd(vpnName);
1389             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1390                     interfaceName, dpnId, vpnName, primaryRd);
1391             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1392                     && !adjacencies.get().getAdjacency().isEmpty()) {
1393                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1394                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1395                         interfaceName, dpnId, vpnName, nextHops);
1396                 for (Adjacency nextHop : nextHops) {
1397                     if (nextHop.isPhysNetworkFunc()) {
1398                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1399                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1400                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1401                                 null/*writeCfgTxn*/);
1402                     } else {
1403                         String rd = nextHop.getVrfId();
1404                         List<String> nhList;
1405                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1406                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1407                             isNonPrimaryAdjIp = Boolean.TRUE;
1408                         } else {
1409                             // This is a primary adjacency
1410                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1411                                     : emptyList();
1412                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1413                                     interfaceName, writeInvTxn);
1414                         }
1415                         if (!nhList.isEmpty()) {
1416                             if (Objects.equals(primaryRd, vpnName)) {
1417                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1418                                 //remove from FIB directly
1419                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1420                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1421                             } else {
1422                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1423                                         interfaceName, writeConfigTxn, writeOperTxn);
1424                             }
1425                         } else {
1426                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1427                                             + " interface {}", nextHop.getIpAddress(), rd,
1428                                     nextHop.getAdjacencyType().toString(), interfaceName);
1429                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1430                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1431                         }
1432                     }
1433                     String ip = nextHop.getIpAddress().split("/")[0];
1434                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1435                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1436                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1437                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1438                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1439                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1440                     }
1441                     // Remove the MIP-IP from VpnPortIpToPort.
1442                     if (isNonPrimaryAdjIp) {
1443                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1444                         if (persistedIp != null && persistedIp.isLearntIp()
1445                                 && persistedIp.getPortName().equals(interfaceName)) {
1446                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1447                             LOG.info(
1448                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1449                                             + "from VpnPortipToPort",
1450                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1451                         }
1452                     }
1453                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1454                     if (vpnPortipToPort != null) {
1455                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1456                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1457                                  + "Interface {} ip {} on dpn {} for vpn {}",
1458                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1459                     }
1460                 }
1461             } else {
1462                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1463                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1464                         + " Removing it.", interfaceName, vpnName, dpnId);
1465                 writeOperTxn.delete(identifier);
1466             }
1467         } catch (ReadFailedException e) {
1468             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1469                     interfaceName, dpnId, vpnName);
1470         }
1471     }
1472
1473     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1474                                                             String interfaceName, BigInteger dpnId,
1475                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1476                                                             TypedWriteTransaction<Operational> writeOperTx) {
1477         return (nh) -> {
1478             String primaryRd = vpnUtil.getVpnRd(vpnName);
1479             String prefix = nextHop.getIpAddress();
1480             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1481             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1482                     nextHop, vpnName, interfaceName, dpnId);
1483             synchronized (vpnNamePrefixKey.intern()) {
1484                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1485                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1486                     //If extra-route is present behind at least one VM, then do not remove or update
1487                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1488                     //prefixtointerface DS
1489                     return;
1490                 }
1491                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1492                         writeConfigTxn);
1493             }
1494             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1495                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1496                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1497         };
1498     }
1499
1500     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1501                                            BigInteger dpnId, String rd, String interfaceName,
1502                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1503                                            TypedWriteTransaction<Operational> writeOperTx) {
1504         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1505                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1506         nhList.forEach((nh) -> {
1507             //IRT: remove routes from other vpns importing it
1508             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1509                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1510             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1511                 String vpnRd = vpn.getVrfId();
1512                 if (vpnRd != null) {
1513                     fibManager.removeOrUpdateFibEntry(vpnRd,
1514                             nextHop.getIpAddress(), nh, writeConfigTxn);
1515                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1516                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1517                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1518                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1519                 }
1520             }
1521         });
1522     }
1523
1524     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1525                                                  int lportTag, String gwMac, String interfaceName,
1526                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1527             throws ExecutionException, InterruptedException {
1528         final Uuid subnetId = nextHop.getSubnetId();
1529         if (nextHop.getSubnetGatewayMacAddress() == null) {
1530             // A valid mac-address was not available for this subnet-gateway-ip
1531             // So a connected-mac-address was used for this subnet and we need
1532             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1533             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1534         }
1535         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1536                 subnetId);
1537     }
1538
1539     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1540                                                   String interfaceName) {
1541         // This is either an extra-route (or) a learned IP via subnet-route
1542         List<String> nhList = null;
1543         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1544         if (nextHopIp == null || nextHopIp.isEmpty()) {
1545             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1546                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1547                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1548                     vpnName);
1549             nhList = emptyList();
1550         } else {
1551             nhList = Collections.singletonList(nextHopIp);
1552         }
1553         return nhList;
1554     }
1555
1556     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1557         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1558         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1559         // else use connected interface
1560         if (gwPort != null && gwPort.isSubnetIp()) {
1561             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1562                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1563             return Optional.of(gwPort.getMacAddress());
1564         }
1565         return Optional.absent();
1566     }
1567
1568     @Override
1569     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1570         final VpnInterface update) {
1571         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1572         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1573                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1574         final String vpnInterfaceName = update.getName();
1575         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1576         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1577         //handles switching between <internal VPN - external VPN>
1578         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1579             List<ListenableFuture<Void>> futures = new ArrayList<>();
1580             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1581                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1582                                 + "to newVpn(s) {}",
1583                         original.getName(), dpnId,
1584                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1585                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1586                 return emptyList();
1587             }
1588             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1589             return futures;
1590         });
1591     }
1592
1593     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1594                                                            VpnInterface original, VpnInterface update,
1595                                                            List<ListenableFuture<Void>> futures) {
1596         boolean isVpnInstanceUpdate = false;
1597         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1598         final String interfaceName = key.getName();
1599         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1600         List<String> oldVpnListCopy = new ArrayList<>();
1601         oldVpnListCopy.addAll(oldVpnList);
1602         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1603         List<String> newVpnListCopy = new ArrayList<>();
1604         newVpnListCopy.addAll(newVpnList);
1605
1606         oldVpnList.removeAll(newVpnList);
1607         newVpnList.removeAll(oldVpnListCopy);
1608         //This block will execute only on if there is a change in the VPN Instance.
1609         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1610             /*
1611              * Internet BGP-VPN Instance update with single router:
1612              * ====================================================
1613              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1614              *     1st VPN Instance : router VPN or external BGP-VPN.
1615              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1616              *
1617              * VPN Instance UPDATE:
1618              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1619              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1620              *
1621              * External BGP-VPN Instance update with single router:
1622              * ====================================================
1623              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1624              *
1625              * Updated VPN Instance will be always either internal router VPN to
1626              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1627              *
1628              * VPN Instance UPDATE:
1629              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1630              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1631              *
1632              * Dual Router VPN Instance Update:
1633              * ================================
1634              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1635              *
1636              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1637              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1638              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1639              *
1640              * Dual Router --> Associated with common external BGP-VPN Instance.
1641              * 1st router and 2nd router are getting associated with single External BGP-VPN
1642              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1643              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1644              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1645              *
1646              *
1647              */
1648             isVpnInstanceUpdate = true;
1649             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1650                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1651                         && (oldVpnList.size() == 1 && newVpnList.size() == 0)) {
1652                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1653                     List<String> externalBgpVpnList = new ArrayList<>();
1654                     for (String newVpnName : newVpnListCopy) {
1655                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1656                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1657                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1658                                 .BgpvpnType.BGPVPNExternal) {
1659                             externalBgpVpnList.add(newVpnName);
1660                             break;
1661                         }
1662                     }
1663                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1664                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1665                             externalBgpVpnList, oldVpnListCopy, futures);
1666
1667                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1668                         && (oldVpnList.size() == 0 && newVpnList.size() == 1)) {
1669                     //Identify the router VPN Instance and pass that value as oldVpnList
1670                     List<String> routerVpnList = new ArrayList<>();
1671                     for (String newVpnName : newVpnListCopy) {
1672                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1673                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1674                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1675                                 .BgpvpnType.VPN) {
1676                             routerVpnList.add(newVpnName);
1677                             break;
1678                         }
1679                     }
1680                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1681                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1682                             newVpnList, oldVpnListCopy, futures);
1683
1684                 } else {
1685                     //Handle remaining use cases.
1686                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1687                             oldVpnListCopy, futures);
1688                 }
1689             } else {
1690                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1691                         oldVpnListCopy, futures);
1692             }
1693         }
1694         return isVpnInstanceUpdate;
1695     }
1696
1697     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1698                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1699                                          List<String> newVpnList, List<String> oldVpnListCopy,
1700                                          List<ListenableFuture<Void>> futures) {
1701         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1702         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1703                 ? origAdjs.getAdjacency() : new ArrayList<>();
1704         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1705         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1706                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1707
1708         boolean isOldVpnRemoveCallExecuted = false;
1709         for (String oldVpnName : oldVpnList) {
1710             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1711                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1712             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1713             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1714                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1715                     newVpnList);
1716             isOldVpnRemoveCallExecuted = true;
1717         }
1718         //Wait for previous interface bindings to be removed
1719         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1720             try {
1721                 Thread.sleep(2000);
1722             } catch (InterruptedException e) {
1723                 //Ignore
1724             }
1725         }
1726         for (String newVpnName : newVpnList) {
1727             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1728             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1729                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1730                         + "onto vpnName {} ", interfaceName, newVpnName);
1731                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1732                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1733                                 + "from oldVpn(s) {} to newVpn {} ",
1734                         interfaceName, oldVpnListCopy, newVpnName);
1735                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1736                  * Use Case:
1737                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1738                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1739                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1740                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1741                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1742                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1743                  *     router VPN as well as Internet BGP-VPN.
1744                  *
1745                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1746                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1747                  *                                       update in existing router VPN instance.
1748                  */
1749                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1750                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1751                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1752                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1753                 }
1754             }
1755         }
1756     }
1757
1758     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1759                                                                     String vpnInterfaceName,
1760                                                                     List<ListenableFuture<Void>> futures) {
1761         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1762         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1763                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1764         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1765         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1766                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1767
1768         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1769         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(update.getVpnInstanceNames(),
1770                 Collections.<VpnInstanceNames>emptyList())) {
1771             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1772             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1773             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1774             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1775             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1776                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1777                 //set of prefix used as entry in prefix-to-interface datastore
1778                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1779                 // in bucket actions on bgp-vpn delete
1780                 Set<String> prefixListForRefreshFib = new HashSet<>();
1781                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1782                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1783                         operTx -> {
1784                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1785                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1786                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1787                                     update.getName(), newVpnName);
1788                             //handle both addition and removal of adjacencies
1789                             // currently, new adjacency may be an extra route
1790                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1791                             if (!oldAdjs.equals(newAdjs)) {
1792                                 for (Adjacency adj : copyNewAdjs) {
1793                                     if (copyOldAdjs.contains(adj)) {
1794                                         copyOldAdjs.remove(adj);
1795                                     } else {
1796                                         // add new adjacency
1797                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1798                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1799                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1800                                         }
1801                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1802                                             + " added to vpn interface {} on vpn {} dpnId {}",
1803                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1804                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1805                                     }
1806                                 }
1807                                 for (Adjacency adj : copyOldAdjs) {
1808                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1809                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1810                                             && !adj.isPhysNetworkFunc()) {
1811                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1812                                                 confTx);
1813                                             //remove FIB entry
1814                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1815                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1816                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1817                                             //remove BGP entry
1818                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1819                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1820                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1821                                             }
1822                                         } else {
1823                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1824                                                 operTx, confTx);
1825                                         }
1826                                     }
1827                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1828                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1829                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1830                                 }
1831                             }
1832                         })));
1833                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1834                     MoreExecutors.directExecutor());
1835                 futures.add(configTxFuture);
1836                 for (ListenableFuture<Void> future : futures) {
1837                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1838                             update.getName(), update.getVpnInstanceNames());
1839                 }
1840             } else {
1841                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1842                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1843             }
1844         }
1845         return futures;
1846     }
1847
1848     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1849         try {
1850             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1851             synchronized (label.toString().intern()) {
1852                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1853                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1854                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1855                         LogicalDatastoreType.OPERATIONAL, lriIid);
1856                 if (opResult.isPresent()) {
1857                     LabelRouteInfo labelRouteInfo =
1858                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1859                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1860                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1861                 }
1862             }
1863             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1864                     nextHopIpList);
1865         } catch (ReadFailedException e) {
1866             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1867                     nextHopIpList);
1868         } catch (TransactionCommitFailedException e) {
1869             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1870                     nextHopIpList);
1871         }
1872     }
1873
1874     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1875         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1876
1877         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1878         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1879                 .addAugmentation(SubnetRoute.class, route).build();
1880         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1881         InstanceIdentifierBuilder<VrfTables> idBuilder =
1882             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1883         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1884         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1885         if (writeConfigTxn != null) {
1886             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1887         } else {
1888             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1889         }
1890         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1891                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1892     }
1893
1894     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1895                                            Adjacency adj, BigInteger dpnId,
1896                                            TypedWriteTransaction<Operational> writeOperTxn,
1897                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1898                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1899                                            Set<String> prefixListForRefreshFib)
1900             throws ExecutionException, InterruptedException {
1901         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1902         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1903         try {
1904             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1905                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1906             if (optVpnInterface.isPresent()) {
1907                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1908                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1909                 String vpnName = currVpnIntf.getVpnInstanceName();
1910                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1911                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1912                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1913                         LogicalDatastoreType.OPERATIONAL, adjPath);
1914                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1915                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1916                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1917                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1918                 List<Adjacency> adjacencies = new ArrayList<>();
1919                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1920                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1921                 }
1922                 long vpnId = vpnUtil.getVpnId(vpnName);
1923                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1924                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1925                 Adjacency operationalAdjacency = null;
1926                 //Handling dual stack neutron port primary adjacency
1927                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1928                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1929                             currVpnIntf.getName(), vpnName);
1930                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1931                             currVpnIntf.getName());
1932                     if (interfaceState != null) {
1933                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1934                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1935                             prefixListForRefreshFib);
1936                     }
1937                 }
1938                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1939                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1940                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1941                             : RouteOrigin.STATIC;
1942                     String nh = adj.getNextHopIpList().get(0);
1943                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1944                     synchronized (vpnPrefixKey.intern()) {
1945                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1946                                 vpnId, null, prefix, vpnName, nh, dpnId);
1947                         if (rdToAllocate.isPresent()) {
1948                             input.setRd(rdToAllocate.get());
1949                             operationalAdjacency = populator.createOperationalAdjacency(input);
1950                             int label = operationalAdjacency.getLabel().intValue();
1951                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1952                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1953                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1954                                     prefixListForRefreshFib, writeConfigTxn);
1955                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1956                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1957                                     vpnName, label, currVpnIntf.getName(), dpnId);
1958                         } else {
1959                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1960                                     vpnName, prefix);
1961                             return;
1962                         }
1963                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1964                         // Keeping the MPLS check for now.
1965                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1966                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1967                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1968                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1969                             vpnsToImportRoute.forEach(vpn -> {
1970                                 if (vpn.getVrfId() != null) {
1971                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1972                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1973                                             .ifPresent(
1974                                                 rds -> vpnManager.addExtraRoute(
1975                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
1976                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
1977                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
1978                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
1979                                 }
1980                             });
1981                         }
1982                     }
1983                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1984                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1985                             currVpnIntf.getName(), vpnName);
1986
1987                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1988                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1989                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1990                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1991                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1992                             Prefixes.PrefixCue.PhysNetFunc);
1993                     if (vpnIntefaceConfig.isPresent()) {
1994                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1995                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1996                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1997                     }
1998
1999                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2000
2001                     writeOperTxn.merge(
2002                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2003                                     prefix), pnfPrefix, true);
2004
2005                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2006                             adj.getIpAddress(), emptyList(), null /* EncapType */, 0 /* label */,
2007                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
2008
2009                     input.setRd(adj.getVrfId());
2010                 }
2011                 if (operationalAdjacency == null) {
2012                     operationalAdjacency = populator.createOperationalAdjacency(input);
2013                 }
2014                 adjacencies.add(operationalAdjacency);
2015                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2016                 VpnInterfaceOpDataEntry newVpnIntf =
2017                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2018                                 aug, dpnId, currVpnIntf.getLportTag(),
2019                                 currVpnIntf.getGatewayMacAddress());
2020
2021                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2022             }
2023         } catch (ReadFailedException e) {
2024             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2025                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2026         }
2027     }
2028
2029     @Nullable
2030     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2031         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2032         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2033     }
2034
2035     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2036                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2037                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2038         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2039         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2040         try {
2041             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2042                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2043             if (optVpnInterface.isPresent()) {
2044                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2045                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2046                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2047                         LogicalDatastoreType.OPERATIONAL, path);
2048                 if (optAdjacencies.isPresent()) {
2049                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2050
2051                     if (adjacencies != null && !adjacencies.isEmpty()) {
2052                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2053                         for (Adjacency adjacency : adjacencies) {
2054                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2055                                 String rd = adjacency.getVrfId();
2056                                 if (adj.getNextHopIpList() != null) {
2057                                     for (String nh : adj.getNextHopIpList()) {
2058                                         deleteExtraRouteFromCurrentAndImportingVpns(
2059                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2060                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2061                                     }
2062                                 } else if (adj.isPhysNetworkFunc()) {
2063                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2064                                             adj.getIpAddress(), adj.getSubnetId());
2065                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2066                                             writeConfigTxn);
2067                                 }
2068                                 break;
2069                             }
2070
2071                         }
2072                     }
2073                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2074                             dpnId, adj.getVrfId());
2075                 } else {
2076                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2077                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2078                 }
2079             }
2080         } catch (ReadFailedException e) {
2081             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2082                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2083         }
2084     }
2085
2086     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2087                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2088                                     TypedWriteTransaction<Operational> writeOperTx) {
2089         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2090         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2091         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2092             String vpnRd = vpn.getVrfId();
2093             if (vpnRd != null) {
2094                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2095                         writeOperTx);
2096             }
2097         }
2098     }
2099
2100     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2101         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2102             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2103             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2104     }
2105
2106     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2107         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2108             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2109     }
2110
2111     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2112                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2113         if (vpnInterface == null) {
2114             return;
2115         }
2116         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2117         if (adjs == null) {
2118             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2119                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2120             return;
2121         }
2122         for (Adjacency adj : adjs) {
2123             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2124                 String primaryInterfaceIp = adj.getIpAddress();
2125                 String macAddress = adj.getMacAddress();
2126                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2127
2128                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2129                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2130
2131                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2132                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2133                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2134                         routerInt, label, writeConfigTxn);
2135                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2136                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2137                         macAddress);
2138             } else {
2139                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2140                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2141                         interfaceName, primaryRd, vpnName);
2142             }
2143         }
2144     }
2145
2146     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2147             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2148         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2149         String rd = vpnUtil.getVpnRd(vpnName);
2150         if (adjs != null) {
2151             List<Adjacency> adjsList = requireNonNullElse(adjs.getAdjacency(), emptyList());
2152             for (Adjacency adj : adjsList) {
2153                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2154                     String primaryInterfaceIp = adj.getIpAddress();
2155                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2156                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2157                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2158                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2159                     return;
2160                 }
2161             }
2162         } else {
2163             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2164                     vpnInterface.getName(), rd);
2165         }
2166     }
2167
2168     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2169         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2170         final String interfaceName = key.getName();
2171         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2172                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2173         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2174                   intefaceData.identifier, vpnName);
2175     }
2176
2177     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2178                                               VpnInterface vpnInterface, String vpnName) {
2179         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2180                .get(vpnName);
2181         if (vpnInterfaces == null) {
2182             vpnInterfaces = new ConcurrentLinkedQueue<>();
2183         }
2184         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2185         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2186         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2187                 vpnInterface.getName(), vpnName);
2188     }
2189
2190     public boolean isVpnInstanceReady(String vpnInstanceName) {
2191         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2192         if (vpnRd == null) {
2193             return false;
2194         }
2195         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2196
2197         return vpnInstanceOpDataEntry != null;
2198     }
2199
2200     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2201         synchronized (vpnInstanceName.intern()) {
2202             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2203                     unprocessedVpnInterfaces.get(vpnInstanceName);
2204             if (vpnInterfaces != null) {
2205                 while (!vpnInterfaces.isEmpty()) {
2206                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2207                     if (hasVpnInstanceCreatedSuccessfully) {
2208                         processSavedInterface(savedInterface, vpnInstanceName);
2209                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2210                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2211                     } else {
2212                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2213                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2214                     }
2215                 }
2216             } else {
2217                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2218             }
2219         }
2220     }
2221
2222     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2223             VpnInterface vpnInterface) {
2224         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2225             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2226                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2227             if (vpnInterfaces != null) {
2228                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2229                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2230                             + "unprocessed list", vpnInterface.getName(),
2231                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2232                 }
2233             } else {
2234                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2235                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2236             }
2237         }
2238     }
2239
2240     public void vpnInstanceIsReady(String vpnInstanceName) {
2241         processSavedInterfaces(vpnInstanceName, true);
2242     }
2243
2244     public void vpnInstanceFailed(String vpnInstanceName) {
2245         processSavedInterfaces(vpnInstanceName, false);
2246     }
2247
2248     private static class UnprocessedVpnInterfaceData {
2249         InstanceIdentifier<VpnInterface> identifier;
2250         VpnInterface vpnInterface;
2251
2252         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2253             this.identifier = identifier;
2254             this.vpnInterface = vpnInterface;
2255         }
2256
2257         @Override
2258         public int hashCode() {
2259             final int prime = 31;
2260             int result = 1;
2261             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2262             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2263             return result;
2264         }
2265
2266         @Override
2267         public boolean equals(Object obj) {
2268             if (this == obj) {
2269                 return true;
2270             }
2271             if (obj == null) {
2272                 return false;
2273             }
2274             if (getClass() != obj.getClass()) {
2275                 return false;
2276             }
2277             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2278             if (identifier == null) {
2279                 if (other.identifier != null) {
2280                     return false;
2281                 }
2282             } else if (!identifier.equals(other.identifier)) {
2283                 return false;
2284             }
2285             if (vpnInterface == null) {
2286                 if (other.vpnInterface != null) {
2287                     return false;
2288                 }
2289             } else if (!vpnInterface.equals(other.vpnInterface)) {
2290                 return false;
2291             }
2292             return true;
2293         }
2294     }
2295
2296     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2297         String primaryRd = vpnUtil.getVpnRd(vpnName);
2298         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2299         if (vpnInstanceOpData == null) {
2300             return;
2301         }
2302         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2303         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2304             return;
2305         }
2306         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2307         vpnToDpnLists.forEach(vpnToDpnList -> {
2308             if (vpnToDpnList.getVpnInterfaces() == null) {
2309                 return;
2310             }
2311             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2312                 try {
2313                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2314                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2315                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2316                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2317                     if (!vpnInterfaceOptional.isPresent()) {
2318                         return;
2319                     }
2320                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2321                             vpnInterface.getInterfaceName());
2322                     if (configVpnAdjacencies == null) {
2323                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2324                         return;
2325                     }
2326                     List<Adjacency> operationVpnAdjacencies = requireNonNullElse(vpnInterfaceOptional.get()
2327                             .augmentation(AdjacenciesOp.class).getAdjacency(), emptyList());
2328                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2329                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2330                     // in operational DS. These unprocessed adjacencies will be handled below.
2331                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2332                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2333                     configVpnAdjacencies.stream()
2334                         .filter(adjacency -> operationVpnAdjacencies.stream()
2335                                 .noneMatch(operationalAdjacency ->
2336                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2337                         .forEach(adjacency -> {
2338                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2339                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2340                                 () -> {
2341                                     // TODO Deal with sequencing — the config tx must only submitted
2342                                     // if the oper tx goes in
2343                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2344                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2345                                         futures.add(
2346                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2347                                                 //set of prefix used, as entry in prefix-to-interface datastore
2348                                                 // is prerequisite for refresh Fib to avoid race condition leading
2349                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2350                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2351                                                 ListenableFuture<Void> configTxFuture =
2352                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2353                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2354                                                             primaryRd, adjacency, vpnInterfaceOptional.get().getDpnId(),
2355                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2356                                                 Futures.addCallback(configTxFuture,
2357                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2358                                                     MoreExecutors.directExecutor());
2359                                                 futures.add(configTxFuture);
2360                                             }));
2361                                         return futures;
2362                                     } else {
2363                                         return emptyList();
2364                                     }
2365                                 });
2366                         });
2367                 } catch (ReadFailedException e) {
2368                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2369                             vpnName, primaryRd);
2370                 }
2371             });
2372         });
2373     }
2374
2375     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2376         private final String interfaceName;
2377         private final boolean add;
2378         private final String txnDestination;
2379
2380         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2381             this.interfaceName = interfaceName;
2382             this.add = add;
2383             this.txnDestination = transactionDest;
2384         }
2385
2386         @Override
2387         public void onSuccess(Void voidObj) {
2388             if (add) {
2389                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2390                         interfaceName, txnDestination);
2391             } else {
2392                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2393             }
2394         }
2395
2396         @Override
2397         public void onFailure(Throwable throwable) {
2398             if (add) {
2399                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2400                         interfaceName, txnDestination, throwable);
2401             } else {
2402                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2403                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2404             }
2405         }
2406     }
2407
2408     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2409         private final String primaryRd;
2410         private final Set<String> prefixListForRefreshFib;
2411
2412         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2413             this.primaryRd = primaryRd;
2414             this.prefixListForRefreshFib = prefixListForRefreshFib;
2415         }
2416
2417         @Override
2418         public void onSuccess(Void voidObj) {
2419             prefixListForRefreshFib.forEach(prefix -> {
2420                 fibManager.refreshVrfEntry(primaryRd, prefix);
2421             });
2422         }
2423
2424         @Override
2425         public void onFailure(Throwable throwable) {
2426             LOG.debug("write Tx config operation failed {}", throwable);
2427         }
2428     }
2429 }