vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.netvirt.vpnmanager.VpnUtil.requireNonNullElse;
  15
  16 import com.google.common.base.Optional;
  17 import com.google.common.base.Preconditions;
  18 import com.google.common.collect.Iterators;
  19 import com.google.common.util.concurrent.FutureCallback;
  20 import com.google.common.util.concurrent.Futures;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import com.google.common.util.concurrent.MoreExecutors;
  23 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  24 import java.math.BigInteger;
  25 import java.util.ArrayList;
  26 import java.util.Collections;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Objects;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.function.Consumer;
  34 import java.util.function.Predicate;
  35 import java.util.stream.Collectors;
  36 import javax.annotation.Nullable;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  42 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  43 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  44 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  45 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  46 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  47 import org.opendaylight.genius.infra.Datastore.Configuration;
  48 import org.opendaylight.genius.infra.Datastore.Operational;
  49 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  51 import org.opendaylight.genius.infra.TransactionAdapter;
  52 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  53 import org.opendaylight.genius.infra.TypedWriteTransaction;
  54 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  55 import org.opendaylight.genius.mdsalutil.NWUtil;
  56 import org.opendaylight.genius.mdsalutil.NwConstants;
  57 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  58 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  59 import org.opendaylight.infrautils.caches.CacheProvider;
  60 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  61 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  62 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  63 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  64 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  65 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  66 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  67 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  68 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  69 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  70 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  71 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  72 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  73 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  74 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  77 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  78 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 121 import org.slf4j.Logger;
 122 import org.slf4j.LoggerFactory;
 123
 124 @Singleton
 125 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 126
 127     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 128     private static final short DJC_MAX_RETRIES = 3;
 129
 130     private final DataBroker dataBroker;
 131     private final ManagedNewTransactionRunner txRunner;
 132     private final IBgpManager bgpManager;
 133     private final IFibManager fibManager;
 134     private final IMdsalApiManager mdsalManager;
 135     private final IdManagerService idManager;
 136     private final OdlInterfaceRpcService ifaceMgrRpcService;
 137     private final VpnFootprintService vpnFootprintService;
 138     private final IInterfaceManager interfaceManager;
 139     private final IVpnManager vpnManager;
 140     private final ArpResponderHandler arpResponderHandler;
 141     private final JobCoordinator jobCoordinator;
 142     private final VpnUtil vpnUtil;
 143
 144     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 145
 146     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 147             new ConcurrentHashMap<>();
 148
 149     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 150
 151     @Inject
 152     public VpnInterfaceManager(final DataBroker dataBroker,
 153                                final IBgpManager bgpManager,
 154                                final IdManagerService idManager,
 155                                final IMdsalApiManager mdsalManager,
 156                                final IFibManager fibManager,
 157                                final OdlInterfaceRpcService ifaceMgrRpcService,
 158                                final VpnFootprintService vpnFootprintService,
 159                                final IInterfaceManager interfaceManager,
 160                                final IVpnManager vpnManager,
 161                                final ArpResponderHandler arpResponderHandler,
 162                                final JobCoordinator jobCoordinator,
 163                                final CacheProvider cacheProvider,
 164                                final VpnUtil vpnUtil) {
 165         super(VpnInterface.class, VpnInterfaceManager.class);
 166
 167         this.dataBroker = dataBroker;
 168         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 169         this.bgpManager = bgpManager;
 170         this.idManager = idManager;
 171         this.mdsalManager = mdsalManager;
 172         this.fibManager = fibManager;
 173         this.ifaceMgrRpcService = ifaceMgrRpcService;
 174         this.vpnFootprintService = vpnFootprintService;
 175         this.interfaceManager = interfaceManager;
 176         this.vpnManager = vpnManager;
 177         this.arpResponderHandler = arpResponderHandler;
 178         this.jobCoordinator = jobCoordinator;
 179         this.vpnUtil = vpnUtil;
 180
 181         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 182                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 183                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 184     }
 185
 186     public Runnable isNotifyTaskQueued(String intfName) {
 187         return vpnIntfMap.remove(intfName);
 188     }
 189
 190     @PostConstruct
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     @PreDestroy
 198     public void close() {
 199         super.close();
 200         vpnInstanceOpDataEntryCache.close();
 201     }
 202
 203     @Override
 204     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 205         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 206     }
 207
 208     @Override
 209     protected VpnInterfaceManager getDataTreeChangeListener() {
 210         return VpnInterfaceManager.this;
 211     }
 212
 213     @Override
 214     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 215         LOG.info("add: intfName {} onto vpnName {}",
 216                  vpnInterface.getName(),
 217                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 218         addVpnInterface(identifier, vpnInterface, null, null);
 219     }
 220
 221     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 222                           final VpnInterface vpnInterface, String vpnName) {
 223         synchronized (vpnName.intern()) {
 224             if (isVpnInstanceReady(vpnName)) {
 225                 return true;
 226             }
 227             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 228             return false;
 229         }
 230     }
 231
 232     // TODO Clean up the exception handling
 233     @SuppressWarnings("checkstyle:IllegalCatch")
 234     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 235                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 236         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(vpnInterface.getVpnInstanceNames(),
 237                 Collections.<VpnInstanceNames>emptyList())) {
 238             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 239             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 240         }
 241     }
 242
 243     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 244                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 245         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 246         final String interfaceName = key.getName();
 247
 248         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 249             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 250                   vpnName, vpnInterface.getName());
 251             return;
 252         }
 253         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 254              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 255         List<Adjacency> copyOldAdjs = null;
 256         if (oldAdjs != null) {
 257             copyOldAdjs = new ArrayList<>();
 258             copyOldAdjs.addAll(oldAdjs);
 259         }
 260         List<Adjacency> copyNewAdjs = null;
 261         if (newAdjs != null) {
 262             copyNewAdjs = new ArrayList<>();
 263             copyNewAdjs.addAll(newAdjs);
 264         }
 265         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 266     }
 267
 268     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 269                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 270                     final @Nullable List<Adjacency> newAdjs,
 271                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 272         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 273         final String interfaceName = key.getName();
 274         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 275         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 276             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 277             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 278             if (interfaceState != null) {
 279                 try {
 280                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 281                     final int ifIndex = interfaceState.getIfIndex();
 282                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 283                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 284                         // (the inventory tx goes in last)
 285                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 286                         ListenableFuture<Void> confFuture =
 287                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 288                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 289                                     operTx -> futures.add(
 290                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 291                                             LOG.info(
 292                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 293                                                     + " on dpn {}",
 294                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 295                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 296                                                 confTx, operTx, invTx, interfaceState, vpnName);
 297                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 298                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 299                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 300                                                 if (newAdjs != null) {
 301                                                     for (Adjacency adj : newAdjs) {
 302                                                         if (oldAdjs.contains(adj)) {
 303                                                             oldAdjs.remove(adj);
 304                                                         } else {
 305                                                             if (!isBgpVpnInternetVpn
 306                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 307                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 308                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx);
 309                                                             }
 310                                                         }
 311                                                     }
 312                                                 }
 313                                                 for (Adjacency adj : oldAdjs) {
 314                                                     if (!isBgpVpnInternetVpn
 315                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 316                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 317                                                             operTx, confTx);
 318                                                     }
 319                                                 }
 320                                             }
 321                                         })))));
 322                         futures.add(confFuture);
 323                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 324                             MoreExecutors.directExecutor());
 325                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 326                             + " processed successfully", interfaceName, vpnName, dpnId);
 327                         return futures;
 328                     });
 329                 } catch (NumberFormatException | IllegalStateException e) {
 330                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 331                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 332                         vpnName, e);
 333                     return;
 334                 }
 335             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 336                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 337                     () -> {
 338                         ListenableFuture<Void> future =
 339                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 340                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 341                                     confTx, vpnName);
 342                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 343                                     vpnName, vpnInterface.getDpnId());
 344                             });
 345                         ListenableFutures.addErrorLogging(future, LOG,
 346                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 347                         return Collections.singletonList(future);
 348                     });
 349             } else {
 350                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 351                     + " is not available", interfaceName, vpnName);
 352             }
 353         } else {
 354             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 355                     + " as vpn is pending delete", interfaceName, vpnName,
 356                 vpnInterface.getDpnId());
 357         }
 358     }
 359
 360     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 361     // see comments below.
 362     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 363     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 364             final int lportTag, boolean isInterfaceUp,
 365             TypedWriteTransaction<Configuration> writeConfigTxn,
 366             TypedWriteTransaction<Operational> writeOperTxn,
 367             TypedReadWriteTransaction<Configuration> writeInvTxn,
 368             Interface interfaceState,
 369             final String vpnName) throws ExecutionException, InterruptedException {
 370         final String interfaceName = vpnInterface.getName();
 371         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 372                 vpnName);
 373         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 374         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 375         if (!isInterfaceUp) {
 376             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 377                      interfaceName, dpId, vpnName);
 378             long vpnId = vpnUtil.getVpnId(vpnName);
 379             if (vpnId == VpnConstants.INVALID_ID) {
 380                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 381                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 382                         dpId);
 383                 return;
 384             }
 385
 386             boolean waitForVpnInterfaceOpRemoval = false;
 387             if (opVpnInterface != null) {
 388                 String opVpnName = opVpnInterface.getVpnInstanceName();
 389                 String primaryInterfaceIp = null;
 390                 if (Objects.equals(opVpnName, vpnName)) {
 391                     // Please check if the primary VRF Entry does not exist for VPNInterface
 392                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 393                     // back to back
 394                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 395                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 396                     if (adjs == null) {
 397                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 398                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 399                                 vpnName);
 400                         return;
 401                     }
 402                     for (Adjacency adj : adjs) {
 403                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 404                             primaryInterfaceIp = adj.getIpAddress();
 405                             break;
 406                         }
 407                     }
 408                     if (primaryInterfaceIp == null) {
 409                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 410                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 411                                 dpId, vpnName);
 412                         return;
 413                     }
 414                     // Get the rd of the vpn instance
 415                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 416                     if (vrf != null) {
 417                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 418                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 419                         return;
 420                     }
 421                     waitForVpnInterfaceOpRemoval = true;
 422                 } else {
 423                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 424                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 425                 }
 426             }
 427             if (!waitForVpnInterfaceOpRemoval) {
 428                 // Add the VPNInterface and quit
 429                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 430                         null/*ipAddressSourceValuePair*/,
 431                         true /* add */);
 432                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 433                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 434                 if (!isBgpVpnInternetVpn) {
 435                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 436                 }
 437                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 438                         dpId, vpnName);
 439                 if (interfaceManager.isExternalInterface(interfaceName)) {
 440                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 441                         NwConstants.ADD_FLOW);
 442                 }
 443                 return;
 444             }
 445
 446             // FIB didn't get a chance yet to clean up this VPNInterface
 447             // Let us give it a chance here !
 448             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 449                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 450             try {
 451                 Runnable notifyTask = new VpnNotifyTask();
 452                 synchronized (notifyTask) {
 453                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 454                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 455                     // published here while holding the lock on it so this path will hit the wait before notify can be
 456                     // invoked.
 457                     vpnIntfMap.put(interfaceName, notifyTask);
 458                     try {
 459                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 460                     } catch (InterruptedException e) {
 461                         // Ignored
 462                     }
 463                 }
 464             } finally {
 465                 vpnIntfMap.remove(interfaceName);
 466             }
 467
 468             if (opVpnInterface != null) {
 469                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 470                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 471                         dpId, vpnName);
 472                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 473                 return;
 474             }
 475             // VPNInterface got removed, proceed with Add
 476             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 477                     interfaceName, dpId, vpnName);
 478             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 479                     null/*ipAddressSourceValuePair*/,
 480                     true /* add */);
 481             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 482                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 483             if (!isBgpVpnInternetVpn) {
 484                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 485             }
 486             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 487                     + " FIB to clean up", interfaceName, dpId, vpnName);
 488             if (interfaceManager.isExternalInterface(interfaceName)) {
 489                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 490                                lportTag, NwConstants.ADD_FLOW);
 491             }
 492         } else {
 493             try {
 494                 // Interface is retained in the DPN, but its Link Up.
 495                 // Advertise prefixes again for this interface to BGP
 496                 InstanceIdentifier<VpnInterface> identifier =
 497                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 498                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 499                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 500                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 501                 // Perform similar operation as interface add event for extraroutes.
 502                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 503                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 504                         LogicalDatastoreType.CONFIGURATION, path);
 505                 if (!optAdjacencies.isPresent()) {
 506                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 507                     return;
 508                 }
 509                 List<Adjacency> adjacencies = requireNonNullElse(optAdjacencies.get().getAdjacency(), emptyList());
 510                 for (Adjacency adjacency : adjacencies) {
 511                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 512                         continue;
 513                     }
 514                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 515                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 516                         continue;
 517                     }
 518                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 519                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn);
 520                 }
 521             } catch (ReadFailedException e) {
 522                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 523                         interfaceName, vpnName, primaryRd, dpId);
 524             }
 525         }
 526     }
 527
 528     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 529         int lportTag, int addOrRemove) {
 530         Uuid extNetworkId;
 531         try {
 532             // vpn instance of ext-net interface is the network-id
 533             extNetworkId = new Uuid(vpnName);
 534         } catch (IllegalArgumentException e) {
 535             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 536                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 537             return;
 538         }
 539
 540         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 541         if (routerIds == null || routerIds.isEmpty()) {
 542             LOG.info("processExternalVpnInterface: No router is associated with {}."
 543                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 544                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 545             return;
 546         }
 547
 548         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 549                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 550         for (Uuid routerId : routerIds) {
 551             String routerName = routerId.getValue();
 552             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 553             if (Objects.equals(primarySwitch, dpId)) {
 554                 Routers router = vpnUtil.getExternalRouter(routerName);
 555                 if (router != null) {
 556                     if (addOrRemove == NwConstants.ADD_FLOW) {
 557                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 558                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 559                                 dpId, interfaceName, lportTag);
 560                     } else {
 561                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 562                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 563                                 dpId, interfaceName, lportTag);
 564                     }
 565                 } else {
 566                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 567                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 568                             interfaceName, dpId, vpnName);
 569                 }
 570             }
 571         }
 572     }
 573
 574     // TODO Clean up the exception handling
 575     @SuppressWarnings("checkstyle:IllegalCatch")
 576     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 577                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 578                                                   String vpnName, String interfaceName) {
 579         if (rd == null) {
 580             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 581                   interfaceName, dpnId, vpnName);
 582             return;
 583         } else {
 584             if (rd.equals(vpnName)) {
 585                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 586                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 587                 return;
 588             }
 589         }
 590         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 591                 interfaceName, dpnId, vpnName, rd);
 592
 593         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 594         if (nextHopIp == null) {
 595             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 596                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 597                     rd, vpnName);
 598             return;
 599         }
 600
 601         try {
 602             //Read NextHops
 603             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 604             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 605                     LogicalDatastoreType.OPERATIONAL, path);
 606             if (adjacencies.isPresent()) {
 607                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 608                 if (nextHops != null && !nextHops.isEmpty()) {
 609                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 610                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 611                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 612                     long l3vni = vpnInstanceOpData.getL3vni();
 613                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 614                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 615                     for (Adjacency nextHop : nextHops) {
 616                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 617                             continue;
 618                         }
 619                         String gatewayMac = null;
 620                         long label = 0;
 621                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 622                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 623                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 624                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 625                                     .get();
 626                         } else {
 627                             label = nextHop.getLabel();
 628                         }
 629                         try {
 630                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 631                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 632                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 633                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 634                                     gatewayMac);
 635                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 636                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 637                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 638                         } catch (Exception e) {
 639                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 640                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 641                                     interfaceName, dpnId, e);
 642                         }
 643                     }
 644                 }
 645             }
 646         } catch (ReadFailedException e) {
 647             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 648                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 649         }
 650     }
 651
 652     // TODO Clean up the exception handling
 653     @SuppressWarnings("checkstyle:IllegalCatch")
 654     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 655                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 656                         TypedWriteTransaction<Operational> writeOperTx) {
 657         //Read NextHops
 658         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 659         String rd = vpnUtil.getVpnRd(interfaceName);
 660         if (rd == null) {
 661             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 662                 interfaceName, vpnName);
 663             return;
 664         } else {
 665             if (rd.equals(vpnName)) {
 666                 LOG.info(
 667                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 668                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 669                 return;
 670             }
 671         }
 672         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 673                vpnName, rd);
 674         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 675         try {
 676             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 677                     path);
 678         } catch (ReadFailedException e) {
 679             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 680                     interfaceName, vpnName);
 681         }
 682         if (adjacencies.isPresent()) {
 683             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 684
 685             if (nextHops != null && !nextHops.isEmpty()) {
 686                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 687                         nextHops, interfaceName, vpnName, rd);
 688                 for (Adjacency nextHop : nextHops) {
 689                     try {
 690                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 691                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 692                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 693                                     interfaceName, vpnName);
 694                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 695                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 696                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 697                                     interfaceName, vpnName);
 698                         } else {
 699                             // Perform similar operation as interface delete event for extraroutes.
 700                             String allocatedRd = nextHop.getVrfId();
 701                             for (String nh : requireNonNullElse(nextHop.getNextHopIpList(),
 702                                     Collections.<String>emptyList())) {
 703                                 deleteExtraRouteFromCurrentAndImportingVpns(
 704                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 705                                     writeOperTx);
 706                             }
 707                         }
 708                     } catch (Exception e) {
 709                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 710                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 711                     }
 712                 }
 713             }
 714         }
 715     }
 716
 717     @SuppressWarnings("checkstyle:IllegalCatch")
 718     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 719                                                   String primaryRd, String interfaceName, final long vpnId,
 720                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 721                                                   TypedWriteTransaction<Operational> writeOperTxn,
 722                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 723                                                   Interface interfaceState)
 724             throws ExecutionException, InterruptedException {
 725         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 726         // Read NextHops
 727         Optional<VpnInterface> vpnInteface = Optional.absent();
 728         try {
 729             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 730             LogicalDatastoreType.CONFIGURATION, identifier);
 731         } catch (ReadFailedException e) {
 732             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 733                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 734         }
 735         Uuid intfnetworkUuid = null;
 736         NetworkType networkType = null;
 737         Long segmentationId = Long.valueOf(-1);
 738         Adjacencies adjacencies = null;
 739         if (vpnInteface.isPresent()) {
 740             intfnetworkUuid = vpnInteface.get().getNetworkId();
 741             networkType = vpnInteface.get().getNetworkType();
 742             segmentationId = vpnInteface.get().getSegmentationId();
 743             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 744             if (adjacencies == null) {
 745                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 746                         null/*gwMac*/, writeOperTxn);
 747                 return;
 748             }
 749         }
 750         // Get the rd of the vpn instance
 751         String nextHopIp = null;
 752         try {
 753             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 754         } catch (Exception e) {
 755             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 756                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 757         }
 758         List<String> nhList = new ArrayList<>();
 759         if (nextHopIp != null) {
 760             nhList.add(nextHopIp);
 761             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 762                     interfaceName, dpnId, vpnName, nhList);
 763         }
 764         Optional<String> gwMac = Optional.absent();
 765         String vpnInterfaceSubnetGwMacAddress = null;
 766         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 767         Long l3vni = vpnInstanceOpData.getL3vni();
 768         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 769         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 770         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 771         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : emptyList();
 772         List<Adjacency> value = new ArrayList<>();
 773         for (Adjacency nextHop : nextHops) {
 774             String rd = primaryRd;
 775             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 776             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 777                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 778                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 779                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 780                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 781                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 782                 continue;
 783             }
 784             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 785                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 786                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 787                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 788                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 789                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 790
 791                 Prefixes prefixes = (intfnetworkUuid != null)
 792                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 793                             segmentationId, prefixCue) :
 794                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 795                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 796                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 797                 final Uuid subnetId = nextHop.getSubnetId();
 798
 799                 String gatewayIp = nextHop.getSubnetGatewayIp();
 800                 if (gatewayIp == null) {
 801                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 802                     if (gatewayIpOptional.isPresent()) {
 803                         gatewayIp = gatewayIpOptional.get();
 804                     }
 805                 }
 806
 807                 if (gatewayIp != null) {
 808                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 809                     if (gwMac.isPresent()) {
 810                         // A valid mac-address is available for this subnet-gateway-ip
 811                         // Use this for programming ARP_RESPONDER table here.  And save this
 812                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 813                         // to populate L3_GW_MAC_TABLE there.
 814                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 815                                 gatewayIp, gwMac.get());
 816                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 817                     } else {
 818                         // A valid mac-address is not available for this subnet-gateway-ip
 819                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 820                         // Save this connected-mac-address as gateway-mac-address for the
 821                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 822                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 823                         if (gwMac.isPresent()) {
 824                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 825                                     NwConstants.ADD_FLOW, gwMac.get());
 826                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 827                                     gatewayIp, gwMac.get());
 828                         } else {
 829                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 830                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 831                                 + "gwIp {}",
 832                                 subnetId, interfaceName, vpnName, gatewayIp);
 833                         }
 834                     }
 835                 } else {
 836                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 837                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 838                         subnetId, interfaceName, vpnName);
 839                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 840                 }
 841                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 842                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 843             } else {
 844                 //Extra route adjacency
 845                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 846                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 847                 synchronized (vpnPrefixKey.intern()) {
 848                     java.util.Optional<String> rdToAllocate = vpnUtil
 849                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 850                                     nextHop.getNextHopIpList().get(0), dpnId);
 851                     if (rdToAllocate.isPresent()) {
 852                         rd = rdToAllocate.get();
 853                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 854                             rd, prefix);
 855                     } else {
 856                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 857                         continue;
 858                     }
 859                 }
 860                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 861                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 862                         interfaceName, dpnId);
 863             }
 864             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 865             // can be different from the gateway-mac-address within the VRFEntry as the
 866             // gateway-mac-address is a superset.
 867             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 868                     : RouteOrigin.STATIC;
 869             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 870                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 871                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 872             Adjacency operationalAdjacency = null;
 873             try {
 874                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 875             } catch (NullPointerException e) {
 876                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 877                     input, e.getMessage());
 878                 return;
 879             }
 880             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 881                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 882                         vpnName, l3vni, origin,
 883                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 884             }
 885             value.add(operationalAdjacency);
 886         }
 887
 888         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 889         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 890                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 891
 892         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 893                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 894                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 895
 896         for (Adjacency nextHop : aug.getAdjacency()) {
 897             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 898             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 899                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 900                         : RouteOrigin.STATIC;
 901                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 902                 registeredPopulator.populateFib(input, writeConfigTxn);
 903             }
 904         }
 905     }
 906
 907     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 908                                               long lportTag, String gwMac,
 909                                               TypedWriteTransaction<Operational> writeOperTxn) {
 910         VpnInterfaceOpDataEntry opInterface =
 911               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 912         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 913             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 914         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 915         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 916                 interfaceName, dpnId, vpnName);
 917     }
 918
 919     // TODO Clean up the exception handling
 920     @SuppressWarnings("checkstyle:IllegalCatch")
 921     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 922                                            StateTunnelList stateTunnelList,
 923                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 924                                            TypedWriteTransaction<Operational> writeOperTxn) {
 925
 926         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 927         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 928         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 929         List<Adjacency> adjList =
 930             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 931         if (adjList.isEmpty()) {
 932             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 933                     vpnInterface, srcDpnId);
 934             return;
 935         }
 936         String prefix = null;
 937         long label = 0;
 938         List<Adjacency> value = new ArrayList<>();
 939         boolean isNextHopAddReqd = false;
 940         String vpnName = vpnInterface.getVpnInstanceName();
 941         long vpnId = vpnUtil.getVpnId(vpnName);
 942         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 943         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 944                 vpnInterface.getName(), vpnInterface.getDpnId(),
 945                 vpnInterface.getVpnInstanceName(), adjList);
 946         for (Adjacency adj : adjList) {
 947             String rd = adj.getVrfId();
 948             rd = rd != null ? rd : vpnName;
 949             prefix = adj.getIpAddress();
 950             label = adj.getLabel();
 951             List<String> nhList = Collections.singletonList(srcTepIp);
 952             List<String> nextHopList = adj.getNextHopIpList();
 953             // If TEP is added , update the nexthop of primary adjacency.
 954             // Secondary adj nexthop is already pointing to primary adj IP address.
 955             if (nextHopList == null || nextHopList.isEmpty()) {
 956                 isNextHopAddReqd = true;
 957             }
 958
 959             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 960                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 961             } else {
 962                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 963                 if (!vrfEntryOptional.isPresent()) {
 964                     continue;
 965                 }
 966                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 967                 if (!nhList.contains(srcTepIp)) {
 968                     nhList.add(srcTepIp);
 969                     isNextHopAddReqd = true;
 970                 }
 971                 value.add(adj);
 972             }
 973
 974             if (isNextHopAddReqd) {
 975                 updateLabelMapper(label, nhList);
 976                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 977                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 978                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 979                 // Update the VRF entry with nextHop
 980                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 981                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 982
 983                 //Get the list of VPN's importing this route(prefix) .
 984                 // Then update the VRF entry with nhList
 985                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 986                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 987                     String vpnRd = vpn.getVrfId();
 988                     if (vpnRd != null) {
 989                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 990                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 991                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 992                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 993                             vpnInterface.getName(), srcDpnId, vpnName,
 994                             vpn.getVpnInstanceName(), vpnRd);
 995                     }
 996                 }
 997                 // Advertise the prefix to BGP only for external vpn
 998                 // since there is a nexthop change.
 999                 try {
1000                     if (!rd.equalsIgnoreCase(vpnName)) {
1001                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1002                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
1003                                 null /*gatewayMacAddress*/);
1004                     }
1005                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1006                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1007                             srcDpnId, vpnName);
1008                 } catch (Exception ex) {
1009                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1010                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1011                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1012                 }
1013             }
1014         }
1015         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1016         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1017                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1018                 .addAugmentation(AdjacenciesOp.class, aug).build();
1019         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1020                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1021         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1022         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1023                 vpnInterface.getName(), srcDpnId, vpnName);
1024
1025     }
1026
1027     // TODO Clean up the exception handling
1028     @SuppressWarnings("checkstyle:IllegalCatch")
1029     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1030                                               StateTunnelList stateTunnelList,
1031                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1032                                               TypedWriteTransaction<Operational> writeOperTxn) {
1033
1034         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1035         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1036         String prefix = null;
1037         long label = 0;
1038         boolean isNextHopRemoveReqd = false;
1039         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1040         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1041         String vpnName = vpnInterface.getVpnInstanceName();
1042         long vpnId = vpnUtil.getVpnId(vpnName);
1043         String primaryRd = vpnUtil.getVpnRd(vpnName);
1044         if (adjList != null) {
1045             List<Adjacency> value = new ArrayList<>();
1046             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1047                     vpnInterface.getName(), vpnInterface.getDpnId(),
1048                     vpnInterface.getVpnInstanceName(), adjList);
1049             for (Adjacency adj : adjList) {
1050                 List<String> nhList = new ArrayList<>();
1051                 String rd = adj.getVrfId();
1052                 rd = rd != null ? rd : vpnName;
1053                 prefix = adj.getIpAddress();
1054                 List<String> nextHopList = adj.getNextHopIpList();
1055                 label = adj.getLabel();
1056                 if (nextHopList != null && !nextHopList.isEmpty()) {
1057                     isNextHopRemoveReqd = true;
1058                 }
1059                 // If TEP is deleted , remove the nexthop from primary adjacency.
1060                 // Secondary adj nexthop will continue to point to primary adj IP address.
1061                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1062                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1063                 } else {
1064                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1065                     if (!vrfEntryOptional.isPresent()) {
1066                         continue;
1067                     }
1068                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1069                     if (nhList.contains(srcTepIp)) {
1070                         nhList.remove(srcTepIp);
1071                         isNextHopRemoveReqd = true;
1072                     }
1073                     value.add(adj);
1074                 }
1075
1076                 if (isNextHopRemoveReqd) {
1077                     updateLabelMapper(label, nhList);
1078                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1079                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1080                             prefix, nhList, vpnName,
1081                             vpnId, rd, vpnInterface.getName());
1082                     // Update the VRF entry with removed nextHop
1083                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1084                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1085
1086                     //Get the list of VPN's importing this route(prefix) .
1087                     // Then update the VRF entry with nhList
1088                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1089                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1090                         String vpnRd = vpn.getVrfId();
1091                         if (vpnRd != null) {
1092                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1093                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1094                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1095                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1096                                     nhList, label, vpnInterface.getName(), srcDpnId,
1097                                     vpnName,
1098                                     vpn.getVpnInstanceName(), vpnRd);
1099                         }
1100                     }
1101
1102                     // Withdraw prefix from BGP only for external vpn.
1103                     try {
1104                         if (!rd.equalsIgnoreCase(vpnName)) {
1105                             bgpManager.withdrawPrefix(rd, prefix);
1106                         }
1107                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1108                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1109                                 vpnInterface.getName(), srcDpnId,
1110                                 vpnName);
1111                     } catch (Exception ex) {
1112                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1113                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1114                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1115                     }
1116                 }
1117             }
1118             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1119             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1120                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1121                     .addAugmentation(AdjacenciesOp.class, aug).build();
1122             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1123                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1124             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1125             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1126                          vpnInterface.getName(), srcDpnId, vpnName);
1127         }
1128     }
1129
1130     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1131         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1132
1133         String vpnRd = vpnUtil.getVpnRd(vpnName);
1134         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1135         if (vpnInstanceOpDataEntry == null) {
1136             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1137                     + " to check for vpns exporting the routes", vpnName);
1138             return vpnsToExportRoute;
1139         }
1140
1141         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1142             if (input.getVpnInstanceName() == null) {
1143                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1144                         input.getVrfId());
1145                 return false;
1146             }
1147             return !input.getVpnInstanceName().equals(vpnName);
1148         };
1149
1150         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1151             Iterable<String> commonRTs =
1152                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1153                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1154             return Iterators.size(commonRTs.iterator()) > 0;
1155         };
1156
1157         vpnsToExportRoute =
1158                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1159                         Collectors.toList());
1160         return vpnsToExportRoute;
1161     }
1162
1163     // TODO Clean up the exception handling
1164     @SuppressWarnings("checkstyle:IllegalCatch")
1165     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1166         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1167         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1168             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1169             if (vrfEntries != null) {
1170                 ListenableFutures.addErrorLogging(
1171                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1172                         for (VrfEntry vrfEntry : vrfEntries) {
1173                             try {
1174                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1175                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1176                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1177                                             + " is not a controller managed non intervpn link route. Ignoring.",
1178                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1179                                     continue;
1180                                 }
1181                                 String prefix = vrfEntry.getDestPrefix();
1182                                 String gwMac = vrfEntry.getGatewayMacAddress();
1183                                 requireNonNullElse(vrfEntry.getRoutePaths(),
1184                                     Collections.<RoutePaths>emptyList()).forEach(routePath -> {
1185                                         String nh = routePath.getNexthopAddress();
1186                                         int label = routePath.getLabel().intValue();
1187                                         if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1188                                             vrfEntry.getOrigin()))) {
1189                                             LOG.info(
1190                                                 "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1191                                                     + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1192                                                 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1193                                             fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1194                                                 Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1195                                                 0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1196                                                 confTx);
1197                                         } else {
1198                                             LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1199                                                     + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1200                                                 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1201                                             SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1202                                             importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1203                                                 confTx);
1204                                         }
1205                                     });
1206                             } catch (RuntimeException e) {
1207                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1208                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1209                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1210                             }
1211                         }
1212                     }), LOG, "Error handing VPN exporting routes");
1213             } else {
1214                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1215                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1216             }
1217         }
1218     }
1219
1220     @Override
1221     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1222         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1223         final String interfaceName = key.getName();
1224         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(vpnInterface.getVpnInstanceNames(),
1225                 Collections.<VpnInstanceNames>emptyList())) {
1226             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1227             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1228         }
1229     }
1230
1231     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1232                                 final VpnInterface vpnInterface, final String vpnName,
1233                                 final String interfaceName) {
1234         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1235             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1236                 ListenableFuture<Void> future =
1237                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1238                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1239                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1240                     });
1241                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1242                         vpnInterface.getName(), vpnName);
1243                 return Collections.singletonList(future);
1244             }, DJC_MAX_RETRIES);
1245         } else {
1246             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1247             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1248         }
1249     }
1250
1251     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1252     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1253                                            final VpnInterface vpnInterface, final String vpnName,
1254                                            final String interfaceName, final Interface interfaceState) {
1255         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1256                 vpnName, vpnInterface.getDpnId());
1257         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1258         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1259             () -> {
1260                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1261                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1262                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1263                         writeOperTxn -> futures.add(
1264                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1265                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1266                                         interfaceName, vpnName);
1267                                 BigInteger dpId;
1268                                 int ifIndex;
1269                                 String gwMacAddress;
1270                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1271                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1272                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1273                                 try {
1274                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1275                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1276                                 } catch (ReadFailedException e) {
1277                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1278                                             interfaceName, vpnName);
1279                                     return;
1280                                 }
1281                                 if (interfaceState != null) {
1282                                     try {
1283                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1284                                     } catch (NumberFormatException | IllegalStateException e) {
1285                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1286                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1287                                                         + " from vpn interface op data store. ", interfaceName,
1288                                                 vpnInterface.getDpnId(), vpnName, e);
1289                                         dpId = BigInteger.ZERO;
1290                                     }
1291                                     ifIndex = interfaceState.getIfIndex();
1292                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1293                                 } else {
1294                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1295                                             + " from vpn interface op.", interfaceName);
1296                                     if (optVpnInterface.isPresent()) {
1297                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1298                                         dpId = vpnOpInterface.getDpnId();
1299                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1300                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1301                                     } else {
1302                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1303                                                 + " as interfaceState and vpn interface op is not"
1304                                                 + " available", interfaceName, vpnName);
1305                                         return;
1306                                     }
1307                                 }
1308                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1309                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1310                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1311                                 LOG.info(
1312                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1313                                                 + "successfully",
1314                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1315                             })))));
1316                 futures.add(configFuture);
1317                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1318                 return futures;
1319             }, DJC_MAX_RETRIES);
1320     }
1321
1322     protected void processVpnInterfaceDown(BigInteger dpId,
1323                                            String interfaceName,
1324                                            int lportTag,
1325                                            String gwMac,
1326                                            VpnInterfaceOpDataEntry vpnOpInterface,
1327                                            boolean isInterfaceStateDown,
1328                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1329                                            TypedWriteTransaction<Operational> writeOperTxn,
1330                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1331             throws ExecutionException, InterruptedException {
1332         if (vpnOpInterface == null) {
1333             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1334                     + " as it is not available in operational data store", interfaceName, dpId);
1335             return;
1336         }
1337         final String vpnName = vpnOpInterface.getVpnInstanceName();
1338         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1339                                                     interfaceName, vpnName);
1340         if (!isInterfaceStateDown) {
1341             final long vpnId = vpnUtil.getVpnId(vpnName);
1342             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1343             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1344             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1345                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1346             if (interfaceManager.isExternalInterface(interfaceName)) {
1347                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1348                     NwConstants.DEL_FLOW);
1349             }
1350             if (!isBgpVpnInternetVpn) {
1351                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1352             }
1353             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1354                     + " successful", interfaceName, dpId, vpnName);
1355         } else {
1356             // Interface is retained in the DPN, but its Link Down.
1357             // Only withdraw the prefixes for this interface from BGP
1358             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1359         }
1360     }
1361
1362     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1363                                           final String vpnName, final long vpnId, String gwMac,
1364                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1365                                           TypedWriteTransaction<Operational> writeOperTxn,
1366                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1367             throws ExecutionException, InterruptedException {
1368         //Read NextHops
1369         try {
1370             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1371                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1372             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1373             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1374                     LogicalDatastoreType.OPERATIONAL, path);
1375             String primaryRd = vpnUtil.getVpnRd(vpnName);
1376             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1377                     interfaceName, dpnId, vpnName, primaryRd);
1378             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1379                     && !adjacencies.get().getAdjacency().isEmpty()) {
1380                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1381                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1382                         interfaceName, dpnId, vpnName, nextHops);
1383                 for (Adjacency nextHop : nextHops) {
1384                     if (nextHop.isPhysNetworkFunc()) {
1385                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1386                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1387                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1388                                 null/*writeCfgTxn*/);
1389                     } else {
1390                         String rd = nextHop.getVrfId();
1391                         List<String> nhList;
1392                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1393                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1394                         } else {
1395                             // This is a primary adjacency
1396                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1397                                     : emptyList();
1398                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1399                                     interfaceName, writeInvTxn);
1400                         }
1401                         if (!nhList.isEmpty()) {
1402                             if (Objects.equals(rd, vpnName)) {
1403                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1404                                 //remove from FIB directly
1405                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1406                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1407                             } else {
1408                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1409                                         interfaceName, writeConfigTxn, writeOperTxn);
1410                             }
1411                         } else {
1412                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1413                                             + " interface {}", nextHop.getIpAddress(), rd,
1414                                     nextHop.getAdjacencyType().toString(), interfaceName);
1415                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1416                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1417                         }
1418                     }
1419                     String ip = nextHop.getIpAddress().split("/")[0];
1420                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1421                     if (vpnVipToPort != null) {
1422                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1423                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1424                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1425                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1426                     }
1427                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1428                     if (vpnPortipToPort != null) {
1429                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1430                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1431                                  + "Interface {} ip {} on dpn {} for vpn {}",
1432                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1433                     }
1434                 }
1435             } else {
1436                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1437                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1438                         + " Removing it.", interfaceName, vpnName, dpnId);
1439                 writeOperTxn.delete(identifier);
1440             }
1441         } catch (ReadFailedException e) {
1442             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1443                     interfaceName, dpnId, vpnName);
1444         }
1445     }
1446
1447     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1448                                                             String interfaceName, BigInteger dpnId,
1449                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1450                                                             TypedWriteTransaction<Operational> writeOperTx) {
1451         return (nh) -> {
1452             String primaryRd = vpnUtil.getVpnRd(vpnName);
1453             String prefix = nextHop.getIpAddress();
1454             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1455             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1456                     nextHop, vpnName, interfaceName, dpnId);
1457             synchronized (vpnNamePrefixKey.intern()) {
1458                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1459                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1460                     //If extra-route is present behind at least one VM, then do not remove or update
1461                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1462                     //prefixtointerface DS
1463                     return;
1464                 }
1465                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1466                         writeConfigTxn);
1467             }
1468             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1469                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1470                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1471         };
1472     }
1473
1474     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1475                                            BigInteger dpnId, String rd, String interfaceName,
1476                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1477                                            TypedWriteTransaction<Operational> writeOperTx) {
1478         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1479                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1480         nhList.forEach((nh) -> {
1481             //IRT: remove routes from other vpns importing it
1482             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1483                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1484             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1485                 String vpnRd = vpn.getVrfId();
1486                 if (vpnRd != null) {
1487                     fibManager.removeOrUpdateFibEntry(vpnRd,
1488                             nextHop.getIpAddress(), nh, writeConfigTxn);
1489                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1490                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1491                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1492                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1493                 }
1494             }
1495         });
1496     }
1497
1498     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1499                                                  int lportTag, String gwMac, String interfaceName,
1500                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1501             throws ExecutionException, InterruptedException {
1502         final Uuid subnetId = nextHop.getSubnetId();
1503         if (nextHop.getSubnetGatewayMacAddress() == null) {
1504             // A valid mac-address was not available for this subnet-gateway-ip
1505             // So a connected-mac-address was used for this subnet and we need
1506             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1507             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1508         }
1509         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1510                 subnetId);
1511     }
1512
1513     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1514                                                   String interfaceName) {
1515         // This is either an extra-route (or) a learned IP via subnet-route
1516         List<String> nhList = null;
1517         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1518         if (nextHopIp == null || nextHopIp.isEmpty()) {
1519             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1520                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1521                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1522                     vpnName);
1523             nhList = emptyList();
1524         } else {
1525             nhList = Collections.singletonList(nextHopIp);
1526         }
1527         return nhList;
1528     }
1529
1530     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1531         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1532         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1533         // else use connected interface
1534         if (gwPort != null && gwPort.isSubnetIp()) {
1535             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1536                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1537             return Optional.of(gwPort.getMacAddress());
1538         }
1539         return Optional.absent();
1540     }
1541
1542     @Override
1543     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1544         final VpnInterface update) {
1545         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1546                 update.getDpnId(), original.getVpnInstanceNames(),
1547                 update.getVpnInstanceNames());
1548         final String vpnInterfaceName = update.getName();
1549         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1550         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1551         //handles switching between <internal VPN - external VPN>
1552         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1553             List<ListenableFuture<Void>> futures = new ArrayList<>();
1554             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1555                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1556                                 + "to newVpn(s) {}",
1557                         original.getName(), dpnId,
1558                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1559                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1560                 return emptyList();
1561             }
1562             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1563             return futures;
1564         });
1565     }
1566
1567     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1568                                                            VpnInterface original, VpnInterface update,
1569                                                            List<ListenableFuture<Void>> futures) {
1570         boolean isVpnInstanceUpdate = Boolean.FALSE;
1571         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1572         final String interfaceName = key.getName();
1573         List<String> oldVpnList = vpnUtil.getVpnListForVpnInterface(original);
1574         List<String> oldVpnListCopy = new ArrayList<>();
1575         oldVpnListCopy.addAll(oldVpnList);
1576         List<String> newVpnList = vpnUtil.getVpnListForVpnInterface(update);
1577
1578         oldVpnList.removeAll(newVpnList);
1579         newVpnList.removeAll(oldVpnListCopy);
1580         //This block will execute only on if there is a change in the VPN Instance.
1581         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1582             /*
1583              * Internet BGP-VPN Instance update with single router:
1584              * ====================================================
1585              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1586              *     1st VPN Instance : router VPN or external BGP-VPN.
1587              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1588              *
1589              * VPN Instance UPDATE:
1590              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1591              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1592              *
1593              * External BGP-VPN Instance update with single router:
1594              * ====================================================
1595              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1596              *
1597              * Updated VPN Instance will be always either internal router VPN to
1598              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1599              *
1600              * VPN Instance UPDATE:
1601              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1602              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1603              *
1604              * TODO Two router VPN instance update use case will be addressed in separate patch
1605              */
1606             isVpnInstanceUpdate = Boolean.TRUE;
1607             updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1608                     oldVpnListCopy, futures);
1609         }
1610         return isVpnInstanceUpdate;
1611     }
1612
1613     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1614                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1615                                          List<String> newVpnList, List<String> oldVpnListCopy,
1616                                          List<ListenableFuture<Void>> futures) {
1617         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1618         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1619                 ? origAdjs.getAdjacency() : new ArrayList<>();
1620         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1621         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1622                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1623
1624         boolean isOldVpnRemoveCallExecuted = false;
1625         for (String oldVpnName : oldVpnList) {
1626             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1627                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1628             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1629             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1630                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1631                     newVpnList);
1632             isOldVpnRemoveCallExecuted = true;
1633         }
1634         //Wait for previous interface bindings to be removed
1635         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1636             try {
1637                 Thread.sleep(2000);
1638             } catch (InterruptedException e) {
1639                 //Ignore
1640             }
1641         }
1642         for (String newVpnName : newVpnList) {
1643             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1644             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1645                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1646                         + "onto vpnName {} ", interfaceName, newVpnName);
1647                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1648                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1649                                 + "from oldVpn(s) {} to newVpn {} ",
1650                         interfaceName, oldVpnListCopy, newVpnName);
1651                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1652                  * Use Case:
1653                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1654                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1655                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1656                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1657                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1658                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1659                  *     router VPN as well as Internet BGP-VPN.
1660                  *
1661                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1662                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1663                  *                                       update in existing router VPN instance.
1664                  */
1665                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1666                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1667                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1668                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1669                 }
1670             }
1671         }
1672     }
1673
1674     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1675                                                                     String vpnInterfaceName,
1676                                                                     List<ListenableFuture<Void>> futures) {
1677         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1678         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1679                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1680         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1681         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1682                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1683
1684         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1685         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(update.getVpnInstanceNames(),
1686                 Collections.<VpnInstanceNames>emptyList())) {
1687             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1688             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1689             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1690             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1691             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1692                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1693                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1694                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
1695                         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1696                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1697                         LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1698                                 update.getName(), newVpnName);
1699                         //handle both addition and removal of adjacencies
1700                         //currently, new adjacency may be an extra route
1701                         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1702                         if (!oldAdjs.equals(newAdjs)) {
1703                             for (Adjacency adj : copyNewAdjs) {
1704                                 if (copyOldAdjs.contains(adj)) {
1705                                     copyOldAdjs.remove(adj);
1706                                 } else {
1707                                     // add new adjacency
1708                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1709                                         addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1710                                                 dpnId, operTx, confTx, confTx);
1711                                     }
1712                                     LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1713                                                     + " vpn interface {} on vpn {} dpnId {}",
1714                                             adj.getIpAddress(), adj.getNextHopIpList(),
1715                                             adj.getLabel(), adj.getSubnetId(), update.getName(),
1716                                             newVpnName, dpnId);
1717                                 }
1718                             }
1719                             for (Adjacency adj : copyOldAdjs) {
1720                                 if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1721                                     if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1722                                             && !adj.isPhysNetworkFunc()) {
1723                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1724                                                 operTx, confTx);
1725                                         //remove FIB entry
1726                                         String vpnRd = vpnUtil.getVpnRd(newVpnName);
1727                                         LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1728                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1729                                         //remove BGP entry
1730                                         fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1731                                         if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1732                                             bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1733                                         }
1734                                     } else {
1735                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1736                                                 operTx, confTx);
1737                                     }
1738                                 }
1739                                 LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1740                                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1741                                                 .getNextHopIpList(),
1742                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1743                             }
1744                         }
1745                     }));
1746                 }));
1747                 for (ListenableFuture<Void> future : futures) {
1748                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1749                             update.getName(), update.getVpnInstanceNames());
1750                 }
1751             } else {
1752                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1753                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1754             }
1755         }
1756         return futures;
1757     }
1758
1759     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1760         try {
1761             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1762             synchronized (label.toString().intern()) {
1763                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1764                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1765                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1766                         LogicalDatastoreType.OPERATIONAL, lriIid);
1767                 if (opResult.isPresent()) {
1768                     LabelRouteInfo labelRouteInfo =
1769                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1770                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1771                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1772                 }
1773             }
1774             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1775                     nextHopIpList);
1776         } catch (ReadFailedException e) {
1777             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1778                     nextHopIpList);
1779         } catch (TransactionCommitFailedException e) {
1780             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1781                     nextHopIpList);
1782         }
1783     }
1784
1785     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1786         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1787
1788         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1789         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1790                 .addAugmentation(SubnetRoute.class, route).build();
1791         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1792         InstanceIdentifierBuilder<VrfTables> idBuilder =
1793             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1794         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1795         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1796         if (writeConfigTxn != null) {
1797             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1798         } else {
1799             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1800         }
1801         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1802                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1803     }
1804
1805     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1806                                            Adjacency adj, BigInteger dpnId,
1807                                            TypedWriteTransaction<Operational> writeOperTxn,
1808                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1809                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1810             throws ExecutionException, InterruptedException {
1811         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1812         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1813         try {
1814             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1815                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1816             if (optVpnInterface.isPresent()) {
1817                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1818                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1819                 String vpnName = currVpnIntf.getVpnInstanceName();
1820                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1821                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1822                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1823                         LogicalDatastoreType.OPERATIONAL, adjPath);
1824                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1825                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1826                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1827                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1828                 List<Adjacency> adjacencies = new ArrayList<>();
1829                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1830                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1831                 }
1832                 long vpnId = vpnUtil.getVpnId(vpnName);
1833                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1834                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1835                 Adjacency operationalAdjacency = null;
1836                 //Handling dual stack neutron port primary adjacency
1837                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1838                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1839                             currVpnIntf.getName(), vpnName);
1840                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1841                             currVpnIntf.getName());
1842                     if (interfaceState != null) {
1843                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1844                                 currVpnIntf.getName(),
1845                                 vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
1846                     }
1847                 }
1848                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1849                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1850                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1851                             : RouteOrigin.STATIC;
1852                     String nh = adj.getNextHopIpList().get(0);
1853                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1854                     synchronized (vpnPrefixKey.intern()) {
1855                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1856                                 vpnId, null, prefix, vpnName, nh, dpnId);
1857                         if (rdToAllocate.isPresent()) {
1858                             input.setRd(rdToAllocate.get());
1859                             operationalAdjacency = populator.createOperationalAdjacency(input);
1860                             int label = operationalAdjacency.getLabel().intValue();
1861                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1862                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1863                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1864                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1865                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1866                                     vpnName, label, currVpnIntf.getName(), dpnId);
1867                         } else {
1868                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1869                                     vpnName, prefix);
1870                             return;
1871                         }
1872                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1873                         // Keeping the MPLS check for now.
1874                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1875                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1876                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1877                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1878                             vpnsToImportRoute.forEach(vpn -> {
1879                                 if (vpn.getVrfId() != null) {
1880                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1881                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1882                                             .ifPresent(
1883                                                 rds -> vpnManager.addExtraRoute(
1884                                                         vpnUtil.getVpnName(vpn.getVpnId()),
1885                                                         adj.getIpAddress(), nh, rds,
1886                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1887                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1888                                                         opAdjacency, encapType, writeConfigTxn));
1889                                 }
1890                             });
1891                         }
1892                     }
1893                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1894                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1895                             currVpnIntf.getName(), vpnName);
1896
1897                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1898                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1899                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1900                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1901                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1902                             Prefixes.PrefixCue.PhysNetFunc);
1903                     if (vpnIntefaceConfig.isPresent()) {
1904                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1905                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1906                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1907                     }
1908
1909                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1910
1911                     writeOperTxn.merge(
1912                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1913                                     prefix), pnfPrefix, true);
1914
1915                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1916                             adj.getIpAddress(), emptyList(), null /* EncapType */, 0 /* label */,
1917                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1918
1919                     input.setRd(adj.getVrfId());
1920                 }
1921                 if (operationalAdjacency == null) {
1922                     operationalAdjacency = populator.createOperationalAdjacency(input);
1923                 }
1924                 adjacencies.add(operationalAdjacency);
1925                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1926                 VpnInterfaceOpDataEntry newVpnIntf =
1927                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1928                                 aug, dpnId, currVpnIntf.getLportTag(),
1929                                 currVpnIntf.getGatewayMacAddress());
1930
1931                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
1932             }
1933         } catch (ReadFailedException e) {
1934             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1935                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1936         }
1937     }
1938
1939     @Nullable
1940     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1941         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
1942         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1943     }
1944
1945     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1946                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
1947                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
1948         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1949         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1950         try {
1951             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1952                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1953             if (optVpnInterface.isPresent()) {
1954                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1955                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1956                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1957                         LogicalDatastoreType.OPERATIONAL, path);
1958                 if (optAdjacencies.isPresent()) {
1959                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1960
1961                     if (adjacencies != null && !adjacencies.isEmpty()) {
1962                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1963                         for (Adjacency adjacency : adjacencies) {
1964                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
1965                                 String rd = adjacency.getVrfId();
1966                                 if (adj.getNextHopIpList() != null) {
1967                                     for (String nh : adj.getNextHopIpList()) {
1968                                         deleteExtraRouteFromCurrentAndImportingVpns(
1969                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1970                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
1971                                     }
1972                                 } else if (adj.isPhysNetworkFunc()) {
1973                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1974                                             adj.getIpAddress(), adj.getSubnetId());
1975                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1976                                             writeConfigTxn);
1977                                 }
1978                                 break;
1979                             }
1980
1981                         }
1982                     }
1983                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1984                             dpnId, adj.getVrfId());
1985                 } else {
1986                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1987                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1988                 }
1989             }
1990         } catch (ReadFailedException e) {
1991             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1992                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1993         }
1994     }
1995
1996     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1997                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
1998                                     TypedWriteTransaction<Operational> writeOperTx) {
1999         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2000         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2001         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2002             String vpnRd = vpn.getVrfId();
2003             if (vpnRd != null) {
2004                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2005                         writeOperTx);
2006             }
2007         }
2008     }
2009
2010     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2011         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2012             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2013             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2014     }
2015
2016     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2017         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2018             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2019     }
2020
2021     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2022                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2023         if (vpnInterface == null) {
2024             return;
2025         }
2026         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2027         if (adjs == null) {
2028             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2029                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2030             return;
2031         }
2032         for (Adjacency adj : adjs) {
2033             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2034                 String primaryInterfaceIp = adj.getIpAddress();
2035                 String macAddress = adj.getMacAddress();
2036                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2037
2038                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2039                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2040
2041                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2042                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2043                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2044                         routerInt, label, writeConfigTxn);
2045                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2046                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2047                         macAddress);
2048                 return;
2049             }
2050         }
2051         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2052                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
2053                 primaryRd, vpnName);
2054     }
2055
2056     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2057             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2058         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2059         String rd = vpnUtil.getVpnRd(vpnName);
2060         if (adjs != null) {
2061             List<Adjacency> adjsList = requireNonNullElse(adjs.getAdjacency(), emptyList());
2062             for (Adjacency adj : adjsList) {
2063                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2064                     String primaryInterfaceIp = adj.getIpAddress();
2065                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2066                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2067                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2068                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2069                     return;
2070                 }
2071             }
2072         } else {
2073             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2074                     vpnInterface.getName(), rd);
2075         }
2076     }
2077
2078     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2079         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
2080             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
2081                   vpnName, intefaceData.vpnInterface.getName());
2082             return;
2083         }
2084         final VpnInterfaceKey key = intefaceData.identifier
2085                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
2086         final String interfaceName = key.getName();
2087         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2088                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2089         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2090                   intefaceData.identifier, vpnName);
2091     }
2092
2093     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2094                                               VpnInterface vpnInterface, String vpnName) {
2095         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2096                .get(vpnName);
2097         if (vpnInterfaces == null) {
2098             vpnInterfaces = new ConcurrentLinkedQueue<>();
2099         }
2100         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2101         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2102         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2103                 vpnInterface.getName(), vpnName);
2104     }
2105
2106     public boolean isVpnInstanceReady(String vpnInstanceName) {
2107         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2108         if (vpnRd == null) {
2109             return false;
2110         }
2111         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2112
2113         return vpnInstanceOpDataEntry != null;
2114     }
2115
2116     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2117         synchronized (vpnInstanceName.intern()) {
2118             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2119                     unprocessedVpnInterfaces.get(vpnInstanceName);
2120             if (vpnInterfaces != null) {
2121                 while (!vpnInterfaces.isEmpty()) {
2122                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2123                     if (hasVpnInstanceCreatedSuccessfully) {
2124                         processSavedInterface(savedInterface, vpnInstanceName);
2125                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2126                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2127                     } else {
2128                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2129                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2130                     }
2131                 }
2132             } else {
2133                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2134             }
2135         }
2136     }
2137
2138     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2139             VpnInterface vpnInterface) {
2140         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2141             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2142                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2143             if (vpnInterfaces != null) {
2144                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2145                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2146                             + "unprocessed list", vpnInterface.getName(),
2147                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2148                 }
2149             } else {
2150                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2151                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2152             }
2153         }
2154     }
2155
2156     public void vpnInstanceIsReady(String vpnInstanceName) {
2157         processSavedInterfaces(vpnInstanceName, true);
2158     }
2159
2160     public void vpnInstanceFailed(String vpnInstanceName) {
2161         processSavedInterfaces(vpnInstanceName, false);
2162     }
2163
2164     private static class UnprocessedVpnInterfaceData {
2165         InstanceIdentifier<VpnInterface> identifier;
2166         VpnInterface vpnInterface;
2167
2168         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2169             this.identifier = identifier;
2170             this.vpnInterface = vpnInterface;
2171         }
2172
2173         @Override
2174         public int hashCode() {
2175             final int prime = 31;
2176             int result = 1;
2177             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2178             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2179             return result;
2180         }
2181
2182         @Override
2183         public boolean equals(Object obj) {
2184             if (this == obj) {
2185                 return true;
2186             }
2187             if (obj == null) {
2188                 return false;
2189             }
2190             if (getClass() != obj.getClass()) {
2191                 return false;
2192             }
2193             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2194             if (identifier == null) {
2195                 if (other.identifier != null) {
2196                     return false;
2197                 }
2198             } else if (!identifier.equals(other.identifier)) {
2199                 return false;
2200             }
2201             if (vpnInterface == null) {
2202                 if (other.vpnInterface != null) {
2203                     return false;
2204                 }
2205             } else if (!vpnInterface.equals(other.vpnInterface)) {
2206                 return false;
2207             }
2208             return true;
2209         }
2210     }
2211
2212     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2213         String primaryRd = vpnUtil.getVpnRd(vpnName);
2214         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2215         if (vpnInstanceOpData == null) {
2216             return;
2217         }
2218         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2219         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2220             return;
2221         }
2222         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2223         vpnToDpnLists.forEach(vpnToDpnList -> {
2224             if (vpnToDpnList.getVpnInterfaces() == null) {
2225                 return;
2226             }
2227             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2228                 try {
2229                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2230                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2231                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2232                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2233                     if (!vpnInterfaceOptional.isPresent()) {
2234                         return;
2235                     }
2236                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2237                             vpnInterface.getInterfaceName());
2238                     if (configVpnAdjacencies == null) {
2239                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2240                         return;
2241                     }
2242                     List<Adjacency> operationVpnAdjacencies = requireNonNullElse(vpnInterfaceOptional.get()
2243                             .augmentation(AdjacenciesOp.class).getAdjacency(), emptyList());
2244                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2245                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2246                     // in operational DS. These unprocessed adjacencies will be handled below.
2247                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2248                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2249                     configVpnAdjacencies.stream()
2250                         .filter(adjacency -> operationVpnAdjacencies.stream()
2251                                 .noneMatch(operationalAdjacency ->
2252                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2253                         .forEach(adjacency -> {
2254                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2255                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2256                                 () -> {
2257                                     // TODO Deal with sequencing — the config tx must only submitted
2258                                     // if the oper tx goes in
2259                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2260                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2261                                         futures.add(
2262                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx ->
2263                                                 futures.add(
2264                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2265                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2266                                                             primaryRd, adjacency, vpnInterfaceOptional.get()
2267                                                                 .getDpnId(), operTx, confTx, confTx)))));
2268                                         return futures;
2269                                     } else {
2270                                         return emptyList();
2271                                     }
2272                                 });
2273                         });
2274                 } catch (ReadFailedException e) {
2275                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2276                             vpnName, primaryRd);
2277                 }
2278             });
2279         });
2280     }
2281
2282     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2283         private final String interfaceName;
2284         private final boolean add;
2285         private final String txnDestination;
2286
2287         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2288             this.interfaceName = interfaceName;
2289             this.add = add;
2290             this.txnDestination = transactionDest;
2291         }
2292
2293         @Override
2294         public void onSuccess(Void voidObj) {
2295             if (add) {
2296                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2297                         interfaceName, txnDestination);
2298             } else {
2299                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2300             }
2301         }
2302
2303         @Override
2304         public void onFailure(Throwable throwable) {
2305             if (add) {
2306                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2307                         interfaceName, txnDestination, throwable);
2308             } else {
2309                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2310                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2311             }
2312         }
2313     }
2314 }