vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13
  14 import com.google.common.base.Optional;
  15 import com.google.common.base.Preconditions;
  16 import com.google.common.collect.Iterators;
  17 import com.google.common.util.concurrent.FutureCallback;
  18 import com.google.common.util.concurrent.Futures;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import com.google.common.util.concurrent.MoreExecutors;
  21 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  22 import java.math.BigInteger;
  23 import java.util.ArrayList;
  24 import java.util.Collections;
  25 import java.util.Iterator;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Objects;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ConcurrentLinkedQueue;
  31 import java.util.concurrent.ExecutionException;
  32 import java.util.function.Consumer;
  33 import java.util.function.Predicate;
  34 import java.util.stream.Collectors;
  35 import javax.annotation.PostConstruct;
  36 import javax.annotation.PreDestroy;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  41 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  42 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  45 import org.opendaylight.genius.infra.Datastore.Configuration;
  46 import org.opendaylight.genius.infra.Datastore.Operational;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  48 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  49 import org.opendaylight.genius.infra.TransactionAdapter;
  50 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  51 import org.opendaylight.genius.infra.TypedWriteTransaction;
  52 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  53 import org.opendaylight.genius.mdsalutil.NWUtil;
  54 import org.opendaylight.genius.mdsalutil.NwConstants;
  55 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  56 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  57 import org.opendaylight.infrautils.caches.CacheProvider;
  58 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  59 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  60 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  61 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  62 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  63 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  64 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  65 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  66 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  67 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  68 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  69 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  70 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  71 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  72 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  73 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  74 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  75 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  76 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 116 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 117 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 118 import org.slf4j.Logger;
 119 import org.slf4j.LoggerFactory;
 120
 121 @Singleton
 122 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 123
 124     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 125     private static final short DJC_MAX_RETRIES = 3;
 126
 127     private final DataBroker dataBroker;
 128     private final ManagedNewTransactionRunner txRunner;
 129     private final IBgpManager bgpManager;
 130     private final IFibManager fibManager;
 131     private final IMdsalApiManager mdsalManager;
 132     private final IdManagerService idManager;
 133     private final OdlInterfaceRpcService ifaceMgrRpcService;
 134     private final VpnFootprintService vpnFootprintService;
 135     private final IInterfaceManager interfaceManager;
 136     private final IVpnManager vpnManager;
 137     private final ArpResponderHandler arpResponderHandler;
 138     private final JobCoordinator jobCoordinator;
 139     private final VpnUtil vpnUtil;
 140
 141     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 142
 143     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 144             new ConcurrentHashMap<>();
 145
 146     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 147
 148     @Inject
 149     public VpnInterfaceManager(final DataBroker dataBroker,
 150                                final IBgpManager bgpManager,
 151                                final IdManagerService idManager,
 152                                final IMdsalApiManager mdsalManager,
 153                                final IFibManager fibManager,
 154                                final OdlInterfaceRpcService ifaceMgrRpcService,
 155                                final VpnFootprintService vpnFootprintService,
 156                                final IInterfaceManager interfaceManager,
 157                                final IVpnManager vpnManager,
 158                                final ArpResponderHandler arpResponderHandler,
 159                                final JobCoordinator jobCoordinator,
 160                                final CacheProvider cacheProvider,
 161                                final VpnUtil vpnUtil) {
 162         super(VpnInterface.class, VpnInterfaceManager.class);
 163
 164         this.dataBroker = dataBroker;
 165         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 166         this.bgpManager = bgpManager;
 167         this.idManager = idManager;
 168         this.mdsalManager = mdsalManager;
 169         this.fibManager = fibManager;
 170         this.ifaceMgrRpcService = ifaceMgrRpcService;
 171         this.vpnFootprintService = vpnFootprintService;
 172         this.interfaceManager = interfaceManager;
 173         this.vpnManager = vpnManager;
 174         this.arpResponderHandler = arpResponderHandler;
 175         this.jobCoordinator = jobCoordinator;
 176         this.vpnUtil = vpnUtil;
 177
 178         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 179                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 180                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 181     }
 182
 183     public Runnable isNotifyTaskQueued(String intfName) {
 184         return vpnIntfMap.remove(intfName);
 185     }
 186
 187     @PostConstruct
 188     public void start() {
 189         LOG.info("{} start", getClass().getSimpleName());
 190         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 191     }
 192
 193     @Override
 194     @PreDestroy
 195     public void close() {
 196         super.close();
 197         vpnInstanceOpDataEntryCache.close();
 198     }
 199
 200     @Override
 201     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 202         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 203     }
 204
 205     @Override
 206     protected VpnInterfaceManager getDataTreeChangeListener() {
 207         return VpnInterfaceManager.this;
 208     }
 209
 210     @Override
 211     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 212         LOG.info("add: intfName {} onto vpnName {}",
 213                  vpnInterface.getName(),
 214                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 215         addVpnInterface(identifier, vpnInterface, null, null);
 216     }
 217
 218     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 219                           final VpnInterface vpnInterface, String vpnName) {
 220         synchronized (vpnName.intern()) {
 221             if (isVpnInstanceReady(vpnName)) {
 222                 return true;
 223             }
 224             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 225             return false;
 226         }
 227     }
 228
 229     // TODO Clean up the exception handling
 230     @SuppressWarnings("checkstyle:IllegalCatch")
 231     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 232                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 233         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 234             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 235             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 236         }
 237     }
 238
 239     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 240                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 241         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 242         final String interfaceName = key.getName();
 243
 244         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 245             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 246                   vpnName, vpnInterface.getName());
 247             return;
 248         }
 249         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 250              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 251         List<Adjacency> copyOldAdjs = null;
 252         if (oldAdjs != null) {
 253             copyOldAdjs = new ArrayList<>();
 254             copyOldAdjs.addAll(oldAdjs);
 255         }
 256         List<Adjacency> copyNewAdjs = null;
 257         if (newAdjs != null) {
 258             copyNewAdjs = new ArrayList<>();
 259             copyNewAdjs.addAll(newAdjs);
 260         }
 261         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 262     }
 263
 264     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 265                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 266                     final List<Adjacency> newAdjs,
 267                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 268         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 269         final String interfaceName = key.getName();
 270         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 271         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 272             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 273             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 274             if (interfaceState != null) {
 275                 try {
 276                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 277                     final int ifIndex = interfaceState.getIfIndex();
 278                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 279                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 280                         // (the inventory tx goes in last)
 281                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 282                         ListenableFuture<Void> confFuture =
 283                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 284                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 285                                     operTx -> futures.add(
 286                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 287                                             LOG.info(
 288                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 289                                                     + " on dpn {}",
 290                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 291                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 292                                                 confTx, operTx, invTx, interfaceState, vpnName);
 293                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 294                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 295                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 296                                                 if (newAdjs != null) {
 297                                                     for (Adjacency adj : newAdjs) {
 298                                                         if (oldAdjs.contains(adj)) {
 299                                                             oldAdjs.remove(adj);
 300                                                         } else {
 301                                                             if (!isBgpVpnInternetVpn
 302                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 303                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 304                                                                     primaryRd, adj, dpnId, operTx, confTx);
 305                                                             }
 306                                                         }
 307                                                     }
 308                                                 }
 309                                                 for (Adjacency adj : oldAdjs) {
 310                                                     if (!isBgpVpnInternetVpn
 311                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 312                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 313                                                             operTx, confTx);
 314                                                     }
 315                                                 }
 316                                             }
 317                                         })))));
 318                         futures.add(confFuture);
 319                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 320                             MoreExecutors.directExecutor());
 321                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 322                             + " processed successfully", interfaceName, vpnName, dpnId);
 323                         return futures;
 324                     });
 325                 } catch (NumberFormatException | IllegalStateException e) {
 326                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 327                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 328                         vpnName, e);
 329                     return;
 330                 }
 331             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 332                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 333                     () -> {
 334                         ListenableFuture<Void> future =
 335                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 336                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 337                                     confTx, vpnName);
 338                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 339                                     vpnName, vpnInterface.getDpnId());
 340                             });
 341                         ListenableFutures.addErrorLogging(future, LOG,
 342                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 343                         return Collections.singletonList(future);
 344                     });
 345             } else {
 346                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 347                     + " is not available", interfaceName, vpnName);
 348             }
 349         } else {
 350             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 351                     + " as vpn is pending delete", interfaceName, vpnName,
 352                 vpnInterface.getDpnId());
 353         }
 354     }
 355
 356     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 357     // see comments below.
 358     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 359     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 360             final int lportTag, boolean isInterfaceUp,
 361             TypedWriteTransaction<Configuration> writeConfigTxn,
 362             TypedWriteTransaction<Operational> writeOperTxn,
 363             TypedReadWriteTransaction<Configuration> writeInvTxn,
 364             Interface interfaceState,
 365             final String vpnName) throws ExecutionException, InterruptedException {
 366         final String interfaceName = vpnInterface.getName();
 367         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 368                 vpnName);
 369         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 370         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 371         if (!isInterfaceUp) {
 372             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 373                      interfaceName, dpId, vpnName);
 374             long vpnId = vpnUtil.getVpnId(vpnName);
 375             if (vpnId == VpnConstants.INVALID_ID) {
 376                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 377                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 378                         dpId);
 379                 return;
 380             }
 381
 382             boolean waitForVpnInterfaceOpRemoval = false;
 383             if (opVpnInterface != null) {
 384                 String opVpnName = opVpnInterface.getVpnInstanceName();
 385                 String primaryInterfaceIp = null;
 386                 if (opVpnName.equals(vpnName)) {
 387                     // Please check if the primary VRF Entry does not exist for VPNInterface
 388                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 389                     // back to back
 390                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 391                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 392                     if (adjs == null) {
 393                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 394                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 395                                 vpnName);
 396                         return;
 397                     }
 398                     for (Adjacency adj : adjs) {
 399                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 400                             primaryInterfaceIp = adj.getIpAddress();
 401                             break;
 402                         }
 403                     }
 404                     if (primaryInterfaceIp == null) {
 405                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 406                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 407                                 dpId, vpnName);
 408                         return;
 409                     }
 410                     // Get the rd of the vpn instance
 411                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 412                     if (vrf != null) {
 413                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 414                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 415                         return;
 416                     }
 417                     waitForVpnInterfaceOpRemoval = true;
 418                 } else {
 419                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 420                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 421                 }
 422             }
 423             if (!waitForVpnInterfaceOpRemoval) {
 424                 // Add the VPNInterface and quit
 425                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 426                         null/*ipAddressSourceValuePair*/,
 427                         true /* add */);
 428                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 429                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 430                 if (!isBgpVpnInternetVpn) {
 431                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 432                 }
 433                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 434                         dpId, vpnName);
 435                 if (interfaceManager.isExternalInterface(interfaceName)) {
 436                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 437                         NwConstants.ADD_FLOW);
 438                 }
 439                 return;
 440             }
 441
 442             // FIB didn't get a chance yet to clean up this VPNInterface
 443             // Let us give it a chance here !
 444             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 445                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 446             try {
 447                 Runnable notifyTask = new VpnNotifyTask();
 448                 synchronized (notifyTask) {
 449                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 450                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 451                     // published here while holding the lock on it so this path will hit the wait before notify can be
 452                     // invoked.
 453                     vpnIntfMap.put(interfaceName, notifyTask);
 454                     try {
 455                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 456                     } catch (InterruptedException e) {
 457                         // Ignored
 458                     }
 459                 }
 460             } finally {
 461                 vpnIntfMap.remove(interfaceName);
 462             }
 463
 464             if (opVpnInterface != null) {
 465                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 466                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 467                         dpId, vpnName);
 468                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 469                 return;
 470             }
 471             // VPNInterface got removed, proceed with Add
 472             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 473                     interfaceName, dpId, vpnName);
 474             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 475                     null/*ipAddressSourceValuePair*/,
 476                     true /* add */);
 477             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 478                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 479             if (!isBgpVpnInternetVpn) {
 480                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 481             }
 482             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 483                     + " FIB to clean up", interfaceName, dpId, vpnName);
 484             if (interfaceManager.isExternalInterface(interfaceName)) {
 485                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 486                                lportTag, NwConstants.ADD_FLOW);
 487             }
 488         } else {
 489             try {
 490                 // Interface is retained in the DPN, but its Link Up.
 491                 // Advertise prefixes again for this interface to BGP
 492                 InstanceIdentifier<VpnInterface> identifier =
 493                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 494                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 495                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 496                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 497                 // Perform similar operation as interface add event for extraroutes.
 498                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 499                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 500                         LogicalDatastoreType.CONFIGURATION, path);
 501                 if (!optAdjacencies.isPresent()) {
 502                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 503                     return;
 504                 }
 505                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 506                 for (Adjacency adjacency : adjacencies) {
 507                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 508                         continue;
 509                     }
 510                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 511                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 512                         continue;
 513                     }
 514                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 515                             dpId, writeOperTxn, writeConfigTxn);
 516                 }
 517             } catch (ReadFailedException e) {
 518                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 519                         interfaceName, vpnName, primaryRd, dpId);
 520             }
 521         }
 522     }
 523
 524     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 525         int lportTag, int addOrRemove) {
 526         Uuid extNetworkId;
 527         try {
 528             // vpn instance of ext-net interface is the network-id
 529             extNetworkId = new Uuid(vpnName);
 530         } catch (IllegalArgumentException e) {
 531             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 532                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 533             return;
 534         }
 535
 536         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 537         if (routerIds == null || routerIds.isEmpty()) {
 538             LOG.info("processExternalVpnInterface: No router is associated with {}."
 539                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 540                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 541             return;
 542         }
 543
 544         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 545                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 546         for (Uuid routerId : routerIds) {
 547             String routerName = routerId.getValue();
 548             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 549             if (Objects.equals(primarySwitch, dpId)) {
 550                 Routers router = vpnUtil.getExternalRouter(routerName);
 551                 if (router != null) {
 552                     if (addOrRemove == NwConstants.ADD_FLOW) {
 553                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 554                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 555                                 dpId, interfaceName, lportTag);
 556                     } else {
 557                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 558                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 559                                 dpId, interfaceName, lportTag);
 560                     }
 561                 } else {
 562                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 563                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 564                             interfaceName, dpId, vpnName);
 565                 }
 566             }
 567         }
 568     }
 569
 570     // TODO Clean up the exception handling
 571     @SuppressWarnings("checkstyle:IllegalCatch")
 572     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 573                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 574                                                   String vpnName, String interfaceName) {
 575         if (rd == null) {
 576             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 577                   interfaceName, dpnId, vpnName);
 578             return;
 579         } else {
 580             if (rd.equals(vpnName)) {
 581                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 582                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 583                 return;
 584             }
 585         }
 586         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 587                 interfaceName, dpnId, vpnName, rd);
 588
 589         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 590         if (nextHopIp == null) {
 591             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 592                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 593                     rd, vpnName);
 594             return;
 595         }
 596
 597         try {
 598             //Read NextHops
 599             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 600             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 601                     LogicalDatastoreType.OPERATIONAL, path);
 602             if (adjacencies.isPresent()) {
 603                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 604                 if (!nextHops.isEmpty()) {
 605                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 606                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 607                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 608                     long l3vni = vpnInstanceOpData.getL3vni();
 609                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 610                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 611                     for (Adjacency nextHop : nextHops) {
 612                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 613                             continue;
 614                         }
 615                         String gatewayMac = null;
 616                         long label = 0;
 617                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 618                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 619                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 620                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 621                                     .get();
 622                         } else {
 623                             label = nextHop.getLabel();
 624                         }
 625                         try {
 626                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 627                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 628                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 629                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 630                                     gatewayMac);
 631                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 632                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 633                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 634                         } catch (Exception e) {
 635                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 636                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 637                                     interfaceName, dpnId, e);
 638                         }
 639                     }
 640                 }
 641             }
 642         } catch (ReadFailedException e) {
 643             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 644                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 645         }
 646     }
 647
 648     // TODO Clean up the exception handling
 649     @SuppressWarnings("checkstyle:IllegalCatch")
 650     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 651                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 652                         TypedWriteTransaction<Operational> writeOperTx) {
 653         //Read NextHops
 654         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 655         String rd = vpnUtil.getVpnRd(interfaceName);
 656         if (rd == null) {
 657             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 658                 interfaceName, vpnName);
 659             return;
 660         } else {
 661             if (rd.equals(vpnName)) {
 662                 LOG.info(
 663                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 664                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 665                 return;
 666             }
 667         }
 668         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 669                vpnName, rd);
 670         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 671         try {
 672             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 673                     path);
 674         } catch (ReadFailedException e) {
 675             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 676                     interfaceName, vpnName);
 677         }
 678         if (adjacencies.isPresent()) {
 679             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 680
 681             if (!nextHops.isEmpty()) {
 682                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 683                         nextHops, interfaceName, vpnName, rd);
 684                 for (Adjacency nextHop : nextHops) {
 685                     try {
 686                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 687                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 688                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 689                                     interfaceName, vpnName);
 690                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 691                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 692                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 693                                     interfaceName, vpnName);
 694                         } else {
 695                             // Perform similar operation as interface delete event for extraroutes.
 696                             String allocatedRd = nextHop.getVrfId();
 697                             for (String nh : nextHop.getNextHopIpList()) {
 698                                 deleteExtraRouteFromCurrentAndImportingVpns(
 699                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 700                                     writeOperTx);
 701                             }
 702                         }
 703                     } catch (Exception e) {
 704                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 705                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 706                     }
 707                 }
 708             }
 709         }
 710     }
 711
 712     @SuppressWarnings("checkstyle:IllegalCatch")
 713     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 714                                                   String primaryRd, String interfaceName, final long vpnId,
 715                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 716                                                   TypedWriteTransaction<Operational> writeOperTxn,
 717                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 718                                                   Interface interfaceState)
 719             throws ExecutionException, InterruptedException {
 720         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 721         // Read NextHops
 722         Optional<VpnInterface> vpnInteface = Optional.absent();
 723         try {
 724             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 725             LogicalDatastoreType.CONFIGURATION, identifier);
 726         } catch (ReadFailedException e) {
 727             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 728                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 729         }
 730         Uuid intfnetworkUuid = null;
 731         NetworkType networkType = null;
 732         Long segmentationId = Long.valueOf(-1);
 733         Adjacencies adjacencies = null;
 734         if (vpnInteface.isPresent()) {
 735             intfnetworkUuid = vpnInteface.get().getNetworkId();
 736             networkType = vpnInteface.get().getNetworkType();
 737             segmentationId = vpnInteface.get().getSegmentationId();
 738             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 739             if (adjacencies == null) {
 740                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 741                         null/*gwMac*/, writeOperTxn);
 742                 return;
 743             }
 744         }
 745         // Get the rd of the vpn instance
 746         String nextHopIp = null;
 747         try {
 748             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 749         } catch (Exception e) {
 750             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 751                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 752         }
 753         List<String> nhList = new ArrayList<>();
 754         if (nextHopIp != null) {
 755             nhList.add(nextHopIp);
 756             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 757                     interfaceName, dpnId, vpnName, nhList);
 758         }
 759         Optional<String> gwMac = Optional.absent();
 760         String vpnInterfaceSubnetGwMacAddress = null;
 761         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 762         Long l3vni = vpnInstanceOpData.getL3vni();
 763         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 764         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 765         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 766         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : Collections.emptyList();
 767         List<Adjacency> value = new ArrayList<>();
 768         for (Adjacency nextHop : nextHops) {
 769             String rd = primaryRd;
 770             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 771             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 772                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 773                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 774                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 775                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 776                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 777                 continue;
 778             }
 779             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 780                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 781                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 782                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 783                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 784                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 785
 786                 Prefixes prefixes = (intfnetworkUuid != null)
 787                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 788                             segmentationId, prefixCue) :
 789                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 790                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 791                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 792                 final Uuid subnetId = nextHop.getSubnetId();
 793
 794                 String gatewayIp = nextHop.getSubnetGatewayIp();
 795                 if (gatewayIp == null) {
 796                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 797                     if (gatewayIpOptional.isPresent()) {
 798                         gatewayIp = gatewayIpOptional.get();
 799                     }
 800                 }
 801
 802                 if (gatewayIp != null) {
 803                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 804                     if (gwMac.isPresent()) {
 805                         // A valid mac-address is available for this subnet-gateway-ip
 806                         // Use this for programming ARP_RESPONDER table here.  And save this
 807                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 808                         // to populate L3_GW_MAC_TABLE there.
 809                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 810                                 gatewayIp, gwMac.get());
 811                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 812                     } else {
 813                         // A valid mac-address is not available for this subnet-gateway-ip
 814                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 815                         // Save this connected-mac-address as gateway-mac-address for the
 816                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 817                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 818                         if (gwMac.isPresent()) {
 819                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 820                                     NwConstants.ADD_FLOW, gwMac.get());
 821                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 822                                     gatewayIp, gwMac.get());
 823                         } else {
 824                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 825                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 826                                 + "gwIp {}",
 827                                 subnetId, interfaceName, vpnName, gatewayIp);
 828                         }
 829                     }
 830                 } else {
 831                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 832                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 833                         subnetId, interfaceName, vpnName);
 834                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 835                 }
 836                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 837                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 838             } else {
 839                 //Extra route adjacency
 840                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 841                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 842                 synchronized (vpnPrefixKey.intern()) {
 843                     java.util.Optional<String> rdToAllocate = vpnUtil
 844                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 845                                     nextHop.getNextHopIpList().get(0), dpnId);
 846                     if (rdToAllocate.isPresent()) {
 847                         rd = rdToAllocate.get();
 848                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 849                             rd, prefix);
 850                     } else {
 851                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 852                         continue;
 853                     }
 854                 }
 855                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 856                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 857                         interfaceName, dpnId);
 858             }
 859             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 860             // can be different from the gateway-mac-address within the VRFEntry as the
 861             // gateway-mac-address is a superset.
 862             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 863                     : RouteOrigin.STATIC;
 864             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 865                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 866                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 867             Adjacency operationalAdjacency = null;
 868             try {
 869                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 870             } catch (NullPointerException e) {
 871                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 872                     input, e.getMessage());
 873                 return;
 874             }
 875             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 876                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 877                         vpnName, l3vni, origin,
 878                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 879             }
 880             value.add(operationalAdjacency);
 881         }
 882
 883         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 884         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 885                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 886
 887         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 888                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 889                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 890
 891         for (Adjacency nextHop : aug.getAdjacency()) {
 892             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 893             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 894                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 895                         : RouteOrigin.STATIC;
 896                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 897                 registeredPopulator.populateFib(input, writeConfigTxn);
 898             }
 899         }
 900     }
 901
 902     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 903                                               long lportTag, String gwMac,
 904                                               TypedWriteTransaction<Operational> writeOperTxn) {
 905         VpnInterfaceOpDataEntry opInterface =
 906               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 907         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 908             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 909         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 910         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 911                 interfaceName, dpnId, vpnName);
 912     }
 913
 914     // TODO Clean up the exception handling
 915     @SuppressWarnings("checkstyle:IllegalCatch")
 916     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 917                                            StateTunnelList stateTunnelList,
 918                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 919                                            TypedWriteTransaction<Operational> writeOperTxn) {
 920
 921         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 922         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 923         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 924         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 925         if (adjList.isEmpty()) {
 926             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 927                     vpnInterface, srcDpnId);
 928             return;
 929         }
 930         String prefix = null;
 931         long label = 0;
 932         List<Adjacency> value = new ArrayList<>();
 933         boolean isNextHopAddReqd = false;
 934         String vpnName = vpnInterface.getVpnInstanceName();
 935         long vpnId = vpnUtil.getVpnId(vpnName);
 936         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 937         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 938                 vpnInterface.getName(), vpnInterface.getDpnId(),
 939                 vpnInterface.getVpnInstanceName(), adjList);
 940         for (Adjacency adj : adjList) {
 941             String rd = adj.getVrfId();
 942             rd = rd != null ? rd : vpnName;
 943             prefix = adj.getIpAddress();
 944             label = adj.getLabel();
 945             List<String> nhList = Collections.singletonList(srcTepIp);
 946             List<String> nextHopList = adj.getNextHopIpList();
 947             // If TEP is added , update the nexthop of primary adjacency.
 948             // Secondary adj nexthop is already pointing to primary adj IP address.
 949             if (nextHopList != null && !nextHopList.isEmpty()) {
 950                 /* everything right already */
 951             } else {
 952                 isNextHopAddReqd = true;
 953             }
 954
 955             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 956                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 957             } else {
 958                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 959                 if (!vrfEntryOptional.isPresent()) {
 960                     continue;
 961                 }
 962                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 963                 if (!nhList.contains(srcTepIp)) {
 964                     nhList.add(srcTepIp);
 965                     isNextHopAddReqd = true;
 966                 }
 967                 value.add(adj);
 968             }
 969
 970             if (isNextHopAddReqd) {
 971                 updateLabelMapper(label, nhList);
 972                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 973                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 974                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 975                 // Update the VRF entry with nextHop
 976                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 977                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 978
 979                 //Get the list of VPN's importing this route(prefix) .
 980                 // Then update the VRF entry with nhList
 981                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 982                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 983                     String vpnRd = vpn.getVrfId();
 984                     if (vpnRd != null) {
 985                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 986                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 987                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 988                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 989                             vpnInterface.getName(), srcDpnId, vpnName,
 990                             vpn.getVpnInstanceName(), vpnRd);
 991                     }
 992                 }
 993                 // Advertise the prefix to BGP only for external vpn
 994                 // since there is a nexthop change.
 995                 try {
 996                     if (!rd.equalsIgnoreCase(vpnName)) {
 997                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 998                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 999                                 null /*gatewayMacAddress*/);
1000                     }
1001                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1002                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1003                             srcDpnId, vpnName);
1004                 } catch (Exception ex) {
1005                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1006                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1007                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1008                 }
1009             }
1010         }
1011         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1012         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1013                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1014                 .addAugmentation(AdjacenciesOp.class, aug).build();
1015         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1016                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1017         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1018         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1019                 vpnInterface.getName(), srcDpnId, vpnName);
1020
1021     }
1022
1023     // TODO Clean up the exception handling
1024     @SuppressWarnings("checkstyle:IllegalCatch")
1025     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1026                                               StateTunnelList stateTunnelList,
1027                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1028                                               TypedWriteTransaction<Operational> writeOperTxn) {
1029
1030         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1031         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1032         String prefix = null;
1033         long label = 0;
1034         boolean isNextHopRemoveReqd = false;
1035         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1036         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1037         String vpnName = vpnInterface.getVpnInstanceName();
1038         long vpnId = vpnUtil.getVpnId(vpnName);
1039         String primaryRd = vpnUtil.getVpnRd(vpnName);
1040         if (adjList != null) {
1041             List<Adjacency> value = new ArrayList<>();
1042             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1043                     vpnInterface.getName(), vpnInterface.getDpnId(),
1044                     vpnInterface.getVpnInstanceName(), adjList);
1045             for (Adjacency adj : adjList) {
1046                 List<String> nhList = new ArrayList<>();
1047                 String rd = adj.getVrfId();
1048                 rd = rd != null ? rd : vpnName;
1049                 prefix = adj.getIpAddress();
1050                 List<String> nextHopList = adj.getNextHopIpList();
1051                 label = adj.getLabel();
1052                 if (nextHopList != null && !nextHopList.isEmpty()) {
1053                     isNextHopRemoveReqd = true;
1054                 }
1055                 // If TEP is deleted , remove the nexthop from primary adjacency.
1056                 // Secondary adj nexthop will continue to point to primary adj IP address.
1057                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1058                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1059                 } else {
1060                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1061                     if (!vrfEntryOptional.isPresent()) {
1062                         continue;
1063                     }
1064                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1065                     if (nhList.contains(srcTepIp)) {
1066                         nhList.remove(srcTepIp);
1067                         isNextHopRemoveReqd = true;
1068                     }
1069                     value.add(adj);
1070                 }
1071
1072                 if (isNextHopRemoveReqd) {
1073                     updateLabelMapper(label, nhList);
1074                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1075                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1076                             prefix, nhList, vpnName,
1077                             vpnId, rd, vpnInterface.getName());
1078                     // Update the VRF entry with removed nextHop
1079                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1080                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1081
1082                     //Get the list of VPN's importing this route(prefix) .
1083                     // Then update the VRF entry with nhList
1084                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1085                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1086                         String vpnRd = vpn.getVrfId();
1087                         if (vpnRd != null) {
1088                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1089                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1090                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1091                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1092                                     nhList, label, vpnInterface.getName(), srcDpnId,
1093                                     vpnName,
1094                                     vpn.getVpnInstanceName(), vpnRd);
1095                         }
1096                     }
1097
1098                     // Withdraw prefix from BGP only for external vpn.
1099                     try {
1100                         if (!rd.equalsIgnoreCase(vpnName)) {
1101                             bgpManager.withdrawPrefix(rd, prefix);
1102                         }
1103                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1104                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1105                                 vpnInterface.getName(), srcDpnId,
1106                                 vpnName);
1107                     } catch (Exception ex) {
1108                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1109                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1110                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1111                     }
1112                 }
1113             }
1114             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1115             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1116                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1117                     .addAugmentation(AdjacenciesOp.class, aug).build();
1118             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1119                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1120             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1121             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1122                          vpnInterface.getName(), srcDpnId, vpnName);
1123         }
1124     }
1125
1126     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1127         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1128
1129         String vpnRd = vpnUtil.getVpnRd(vpnName);
1130         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1131         if (vpnInstanceOpDataEntry == null) {
1132             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1133                     + " to check for vpns exporting the routes", vpnName);
1134             return vpnsToExportRoute;
1135         }
1136
1137         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1138             if (input.getVpnInstanceName() == null) {
1139                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1140                         input.getVrfId());
1141                 return false;
1142             }
1143             return !input.getVpnInstanceName().equals(vpnName);
1144         };
1145
1146         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1147             Iterable<String> commonRTs =
1148                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1149                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1150             return Iterators.size(commonRTs.iterator()) > 0;
1151         };
1152
1153         vpnsToExportRoute =
1154                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1155                         Collectors.toList());
1156         return vpnsToExportRoute;
1157     }
1158
1159     // TODO Clean up the exception handling
1160     @SuppressWarnings("checkstyle:IllegalCatch")
1161     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1162         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1163         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1164             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1165             if (vrfEntries != null) {
1166                 ListenableFutures.addErrorLogging(
1167                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1168                         for (VrfEntry vrfEntry : vrfEntries) {
1169                             try {
1170                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1171                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1172                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1173                                             + " is not a controller managed non intervpn link route. Ignoring.",
1174                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1175                                     continue;
1176                                 }
1177                                 String prefix = vrfEntry.getDestPrefix();
1178                                 String gwMac = vrfEntry.getGatewayMacAddress();
1179                                 vrfEntry.getRoutePaths().forEach(routePath -> {
1180                                     String nh = routePath.getNexthopAddress();
1181                                     int label = routePath.getLabel().intValue();
1182                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1183                                         vrfEntry.getOrigin()))) {
1184                                         LOG.info(
1185                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1186                                                 + " nexthop {} label {} to vpn {} vpnRd {}",
1187                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1188                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1189                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1190                                             0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1191                                             confTx);
1192                                     } else {
1193                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1194                                                 + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1195                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1196                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1197                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1198                                             confTx);
1199                                     }
1200                                 });
1201                             } catch (RuntimeException e) {
1202                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1203                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1204                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1205                             }
1206                         }
1207                     }), LOG, "Error handing VPN exporting routes");
1208             } else {
1209                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1210                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1211             }
1212         }
1213     }
1214
1215     @Override
1216     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1217         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1218         final String interfaceName = key.getName();
1219         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1220             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1221             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1222         }
1223     }
1224
1225     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1226                                 final VpnInterface vpnInterface, final String vpnName,
1227                                 final String interfaceName) {
1228         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1229             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1230                 ListenableFuture<Void> future =
1231                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1232                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1233                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1234                     });
1235                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1236                         vpnInterface.getName(), vpnName);
1237                 return Collections.singletonList(future);
1238             }, DJC_MAX_RETRIES);
1239         } else {
1240             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1241             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1242         }
1243     }
1244
1245     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1246     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1247                                            final VpnInterface vpnInterface, final String vpnName,
1248                                            final String interfaceName, final Interface interfaceState) {
1249         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1250                 vpnName, vpnInterface.getDpnId());
1251         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1252         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1253             () -> {
1254                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1255                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1256                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1257                         writeOperTxn -> futures.add(
1258                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1259                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1260                                         interfaceName, vpnName);
1261                                 BigInteger dpId = BigInteger.ZERO;
1262                                 int ifIndex = 0;
1263                                 String gwMacAddress = null;
1264                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1265                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1266                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1267                                 try {
1268                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1269                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1270                                 } catch (ReadFailedException e) {
1271                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1272                                             interfaceName, vpnName);
1273                                     return;
1274                                 }
1275                                 if (interfaceState != null) {
1276                                     try {
1277                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1278                                     } catch (NumberFormatException | IllegalStateException e) {
1279                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1280                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1281                                                         + " from vpn interface op data store. ", interfaceName,
1282                                                 vpnInterface.getDpnId(), vpnName, e);
1283                                         dpId = BigInteger.ZERO;
1284                                     }
1285                                     ifIndex = interfaceState.getIfIndex();
1286                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1287                                 } else {
1288                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1289                                             + " from vpn interface op.", interfaceName);
1290                                     if (optVpnInterface.isPresent()) {
1291                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1292                                         dpId = vpnOpInterface.getDpnId();
1293                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1294                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1295                                     } else {
1296                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1297                                                 + " as interfaceState and vpn interface op is not"
1298                                                 + " available", interfaceName, vpnName);
1299                                         return;
1300                                     }
1301                                 }
1302                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1303                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1304                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1305                                 LOG.info(
1306                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1307                                                 + "successfully",
1308                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1309                             })))));
1310                 futures.add(configFuture);
1311                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1312                 return futures;
1313             }, DJC_MAX_RETRIES);
1314     }
1315
1316     protected void processVpnInterfaceDown(BigInteger dpId,
1317                                            String interfaceName,
1318                                            int lportTag,
1319                                            String gwMac,
1320                                            VpnInterfaceOpDataEntry vpnOpInterface,
1321                                            boolean isInterfaceStateDown,
1322                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1323                                            TypedWriteTransaction<Operational> writeOperTxn,
1324                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1325             throws ExecutionException, InterruptedException {
1326         if (vpnOpInterface == null) {
1327             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1328                     + " as it is not available in operational data store", interfaceName, dpId);
1329             return;
1330         }
1331         final String vpnName = vpnOpInterface.getVpnInstanceName();
1332         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1333                                                     interfaceName, vpnName);
1334         if (!isInterfaceStateDown) {
1335             final long vpnId = vpnUtil.getVpnId(vpnName);
1336             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1337             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1338             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1339                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1340             if (interfaceManager.isExternalInterface(interfaceName)) {
1341                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1342                     NwConstants.DEL_FLOW);
1343             }
1344             if (!isBgpVpnInternetVpn) {
1345                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1346             }
1347             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1348                     + " successful", interfaceName, dpId, vpnName);
1349         } else {
1350             // Interface is retained in the DPN, but its Link Down.
1351             // Only withdraw the prefixes for this interface from BGP
1352             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1353         }
1354     }
1355
1356     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1357                                           final String vpnName, final long vpnId, String gwMac,
1358                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1359                                           TypedWriteTransaction<Operational> writeOperTxn,
1360                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1361             throws ExecutionException, InterruptedException {
1362         //Read NextHops
1363         try {
1364             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1365                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1366             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1367             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1368                     LogicalDatastoreType.OPERATIONAL, path);
1369             String primaryRd = vpnUtil.getVpnRd(vpnName);
1370             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1371                     interfaceName, dpnId, vpnName, primaryRd);
1372             if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1373                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1374                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1375                         interfaceName, dpnId, vpnName, nextHops);
1376                 for (Adjacency nextHop : nextHops) {
1377                     if (nextHop.isPhysNetworkFunc()) {
1378                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1379                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1380                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1381                                 null/*writeCfgTxn*/);
1382                     } else {
1383                         String rd = nextHop.getVrfId();
1384                         List<String> nhList;
1385                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1386                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1387                         } else {
1388                             // This is a primary adjacency
1389                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1390                                     : Collections.emptyList();
1391                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1392                                     interfaceName, writeInvTxn);
1393                         }
1394                         if (!nhList.isEmpty()) {
1395                             if (rd.equals(vpnName)) {
1396                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1397                                 //remove from FIB directly
1398                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1399                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1400                             } else {
1401                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1402                                         interfaceName, writeConfigTxn, writeOperTxn);
1403                             }
1404                         } else {
1405                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1406                                             + " interface {}", nextHop.getIpAddress(), rd,
1407                                     nextHop.getAdjacencyType().toString(), interfaceName);
1408                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1409                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1410                         }
1411                     }
1412                     String ip = nextHop.getIpAddress().split("/")[0];
1413                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1414                     if (vpnVipToPort != null) {
1415                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1416                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1417                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1418                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1419                     }
1420                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1421                     if (vpnPortipToPort != null) {
1422                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1423                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1424                                  + "Interface {} ip {} on dpn {} for vpn {}",
1425                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1426                     }
1427                 }
1428             } else {
1429                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1430                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1431                         + " Removing it.", interfaceName, vpnName, dpnId);
1432                 writeOperTxn.delete(identifier);
1433             }
1434         } catch (ReadFailedException e) {
1435             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1436                     interfaceName, dpnId, vpnName);
1437         }
1438     }
1439
1440     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1441                                                             String interfaceName, BigInteger dpnId,
1442                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1443                                                             TypedWriteTransaction<Operational> writeOperTx) {
1444         return (nh) -> {
1445             String primaryRd = vpnUtil.getVpnRd(vpnName);
1446             String prefix = nextHop.getIpAddress();
1447             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1448             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1449                     nextHop, vpnName, interfaceName, dpnId);
1450             synchronized (vpnNamePrefixKey.intern()) {
1451                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1452                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1453                     //If extra-route is present behind at least one VM, then do not remove or update
1454                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1455                     //prefixtointerface DS
1456                     return;
1457                 }
1458                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1459                         writeConfigTxn);
1460             }
1461             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1462                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1463                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1464         };
1465     }
1466
1467     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1468                                            BigInteger dpnId, String rd, String interfaceName,
1469                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1470                                            TypedWriteTransaction<Operational> writeOperTx) {
1471         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1472                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1473         nhList.forEach((nh) -> {
1474             //IRT: remove routes from other vpns importing it
1475             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1476                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1477             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1478                 String vpnRd = vpn.getVrfId();
1479                 if (vpnRd != null) {
1480                     fibManager.removeOrUpdateFibEntry(vpnRd,
1481                             nextHop.getIpAddress(), nh, writeConfigTxn);
1482                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1483                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1484                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1485                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1486                 }
1487             }
1488         });
1489     }
1490
1491     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1492                                                  int lportTag, String gwMac, String interfaceName,
1493                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1494             throws ExecutionException, InterruptedException {
1495         final Uuid subnetId = nextHop.getSubnetId();
1496         if (nextHop.getSubnetGatewayMacAddress() == null) {
1497             // A valid mac-address was not available for this subnet-gateway-ip
1498             // So a connected-mac-address was used for this subnet and we need
1499             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1500             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1501         }
1502         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1503                 subnetId);
1504     }
1505
1506     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1507                                                   String interfaceName) {
1508         // This is either an extra-route (or) a learned IP via subnet-route
1509         List<String> nhList = null;
1510         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1511         if (nextHopIp == null || nextHopIp.isEmpty()) {
1512             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1513                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1514                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1515                     vpnName);
1516             nhList = Collections.emptyList();
1517         } else {
1518             nhList = Collections.singletonList(nextHopIp);
1519         }
1520         return nhList;
1521     }
1522
1523     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1524         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1525         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1526         // else use connected interface
1527         if (gwPort != null && gwPort.isSubnetIp()) {
1528             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1529                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1530             return Optional.of(gwPort.getMacAddress());
1531         }
1532         return Optional.absent();
1533     }
1534
1535     @Override
1536     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1537         final VpnInterface update) {
1538         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1539                 update.getDpnId(), original.getVpnInstanceNames(),
1540                 update.getVpnInstanceNames());
1541         final String vpnInterfaceName = update.getName();
1542         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1543         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1544         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1545             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1546         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1547         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1548             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1549
1550         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1551         //handles switching between <internal VPN - external VPN>
1552         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1553             if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1554                 LOG.info("update: handled VPNInterface {} on dpn {} update"
1555                                 + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1556                         original.getName(), dpnId,
1557                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1558                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1559                 return Collections.emptyList();
1560             }
1561             List<ListenableFuture<Void>> futures = new ArrayList<>();
1562             for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1563                 String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1564                 List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1565                 List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1566                 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1567                 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1568                     // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1569                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1570                         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
1571                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1572                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1573                             LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1574                                     update.getName(), newVpnName);
1575                             //handle both addition and removal of adjacencies
1576                             //currently, new adjacency may be an extra route
1577                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1578                             if (!oldAdjs.equals(newAdjs)) {
1579                                 for (Adjacency adj : copyNewAdjs) {
1580                                     if (copyOldAdjs.contains(adj)) {
1581                                         copyOldAdjs.remove(adj);
1582                                     } else {
1583                                         // add new adjacency - right now only extra route will hit this path
1584                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1585                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1586                                                     dpnId, operTx, confTx);
1587                                         }
1588                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1589                                                 + " vpn interface {} on vpn {} dpnId {}",
1590                                                 adj.getIpAddress(), adj.getNextHopIpList(),
1591                                                 adj.getLabel(), adj.getSubnetId(), update.getName(),
1592                                                 newVpnName, dpnId);
1593                                     }
1594                                 }
1595                                 for (Adjacency adj : copyOldAdjs) {
1596                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1597                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1598                                                 && !adj.isPhysNetworkFunc()) {
1599                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1600                                                     operTx, confTx);
1601                                             //remove FIB entry
1602                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1603                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1604                                                     + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1605                                             //remove BGP entry
1606                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1607                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1608                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1609                                             }
1610                                         } else {
1611                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1612                                                 operTx, confTx);
1613                                         }
1614                                     }
1615                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1616                                                     + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1617                                                     .getNextHopIpList(),
1618                                             adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1619                                 }
1620                             }
1621                         }));
1622                     }));
1623                     for (ListenableFuture<Void> future : futures) {
1624                         ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1625                                 update.getName(), update.getVpnInstanceNames());
1626                     }
1627                 } else {
1628                     LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1629                             + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1630                 }
1631             }
1632             return futures;
1633         });
1634     }
1635
1636     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1637                                                  VpnInterface original, VpnInterface update) {
1638         boolean isSwap = Boolean.FALSE;
1639         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1640         final String interfaceName = key.getName();
1641         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1642             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1643         List<String> oldVpnListCopy = new ArrayList<>();
1644         oldVpnListCopy.addAll(oldVpnList);
1645         List<String> newVpnList = update.getVpnInstanceNames().stream()
1646             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1647         oldVpnList.removeAll(newVpnList);
1648         newVpnList.removeAll(oldVpnListCopy);
1649         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1650             for (String oldVpnName: oldVpnList) {
1651                 isSwap = Boolean.TRUE;
1652                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1653                         + " running config-driven swap removal", interfaceName, oldVpnName);
1654                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1655                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1656                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1657             }
1658             //Wait for previous interface bindings to be removed
1659             try {
1660                 Thread.sleep(2000);
1661             } catch (InterruptedException e) {
1662                 //Ignore
1663             }
1664
1665             final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1666             final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1667                     ? origAdjs.getAdjacency() : new ArrayList<>();
1668             final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1669             final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1670                     ? updateAdjs.getAdjacency() : new ArrayList<>();
1671             for (String newVpnName: newVpnList) {
1672                 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1673                 isSwap = Boolean.TRUE;
1674                 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1675                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1676                             + "running config-driven swap addition", interfaceName, newVpnName);
1677                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1678                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1679                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1680                             interfaceName, oldVpnListCopy, newVpnName);
1681                 }
1682             }
1683         }
1684         return isSwap;
1685     }
1686
1687     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1688         try {
1689             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1690             synchronized (label.toString().intern()) {
1691                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1692                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1693                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1694                         LogicalDatastoreType.OPERATIONAL, lriIid);
1695                 if (opResult.isPresent()) {
1696                     LabelRouteInfo labelRouteInfo =
1697                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1698                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1699                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1700                 }
1701             }
1702             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1703                     nextHopIpList);
1704         } catch (ReadFailedException e) {
1705             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1706                     nextHopIpList);
1707         } catch (TransactionCommitFailedException e) {
1708             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1709                     nextHopIpList);
1710         }
1711     }
1712
1713     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1714         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1715
1716         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1717         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1718                 .addAugmentation(SubnetRoute.class, route).build();
1719         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1720         InstanceIdentifierBuilder<VrfTables> idBuilder =
1721             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1722         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1723         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1724         if (writeConfigTxn != null) {
1725             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1726         } else {
1727             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1728         }
1729         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1730                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1731     }
1732
1733     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1734                                            Adjacency adj, BigInteger dpnId,
1735                                            TypedWriteTransaction<Operational> writeOperTxn,
1736                                            TypedWriteTransaction<Configuration> writeConfigTxn)
1737             throws ExecutionException, InterruptedException {
1738         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1739         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1740         try {
1741             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1742                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1743             if (optVpnInterface.isPresent()) {
1744                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1745                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1746                 String vpnName = currVpnIntf.getVpnInstanceName();
1747                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1748                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1749                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1750                         LogicalDatastoreType.OPERATIONAL, adjPath);
1751                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1752                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1753                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1754                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1755                 List<Adjacency> adjacencies;
1756                 if (optAdjacencies.isPresent()) {
1757                     adjacencies = optAdjacencies.get().getAdjacency();
1758                 } else {
1759                     // This code will be hit in case of first PNF adjacency
1760                     adjacencies = new ArrayList<>();
1761                 }
1762                 long vpnId = vpnUtil.getVpnId(vpnName);
1763                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1764                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1765                 Adjacency operationalAdjacency = null;
1766                 //Handling dual stack neutron port primary adjacency
1767                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1768                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1769                             currVpnIntf.getName(), vpnName);
1770                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1771                             currVpnIntf.getName());
1772                     if (interfaceState != null) {
1773                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1774                                 currVpnIntf.getName(),
1775                                 vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1776                     }
1777                 }
1778                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1779                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1780                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1781                             : RouteOrigin.STATIC;
1782                     String nh = adj.getNextHopIpList().get(0);
1783                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1784                     synchronized (vpnPrefixKey.intern()) {
1785                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1786                                 vpnId, null, prefix, vpnName, nh, dpnId);
1787                         if (rdToAllocate.isPresent()) {
1788                             input.setRd(rdToAllocate.get());
1789                             operationalAdjacency = populator.createOperationalAdjacency(input);
1790                             int label = operationalAdjacency.getLabel().intValue();
1791                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1792                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1793                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1794                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1795                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1796                                     vpnName, label, currVpnIntf.getName(), dpnId);
1797                         } else {
1798                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1799                                     vpnName, prefix);
1800                             return;
1801                         }
1802                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1803                         // Keeping the MPLS check for now.
1804                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1805                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1806                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1807                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1808                             vpnsToImportRoute.forEach(vpn -> {
1809                                 if (vpn.getVrfId() != null) {
1810                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1811                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1812                                             .ifPresent(
1813                                                 rds -> vpnManager.addExtraRoute(
1814                                                         vpnUtil.getVpnName(vpn.getVpnId()),
1815                                                         adj.getIpAddress(), nh, rds,
1816                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1817                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1818                                                         opAdjacency, encapType, writeConfigTxn));
1819                                 }
1820                             });
1821                         }
1822                     }
1823                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1824                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1825                             currVpnIntf.getName(), vpnName);
1826
1827                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1828                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1829                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1830                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1831                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1832                             Prefixes.PrefixCue.PhysNetFunc);
1833                     if (vpnIntefaceConfig.isPresent()) {
1834                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1835                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1836                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1837                     }
1838
1839                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1840
1841                     writeOperTxn.merge(
1842                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1843                                     prefix), pnfPrefix, true);
1844
1845                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1846                             adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1847                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1848
1849                     input.setRd(adj.getVrfId());
1850                 }
1851                 if (operationalAdjacency == null) {
1852                     operationalAdjacency = populator.createOperationalAdjacency(input);
1853                 }
1854                 adjacencies.add(operationalAdjacency);
1855                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1856                 VpnInterfaceOpDataEntry newVpnIntf =
1857                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1858                                 aug, dpnId, currVpnIntf.getLportTag(),
1859                                 currVpnIntf.getGatewayMacAddress());
1860
1861                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
1862             }
1863         } catch (ReadFailedException e) {
1864             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1865                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1866         }
1867     }
1868
1869     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1870         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
1871         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1872     }
1873
1874     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1875                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
1876                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
1877         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1878         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1879         try {
1880             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1881                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1882             if (optVpnInterface.isPresent()) {
1883                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1884                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1885                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1886                         LogicalDatastoreType.OPERATIONAL, path);
1887                 if (optAdjacencies.isPresent()) {
1888                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1889
1890                     if (!adjacencies.isEmpty()) {
1891                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1892                         Iterator<Adjacency> adjIt = adjacencies.iterator();
1893                         while (adjIt.hasNext()) {
1894                             Adjacency adjElem = adjIt.next();
1895                             if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1896                                 String rd = adjElem.getVrfId();
1897                                 if (adj.getNextHopIpList() != null) {
1898                                     for (String nh : adj.getNextHopIpList()) {
1899                                         deleteExtraRouteFromCurrentAndImportingVpns(
1900                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1901                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
1902                                     }
1903                                 } else if (adj.isPhysNetworkFunc()) {
1904                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1905                                             adj.getIpAddress(), adj.getSubnetId());
1906                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1907                                             writeConfigTxn);
1908                                 }
1909                                 break;
1910                             }
1911
1912                         }
1913                     }
1914                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1915                             dpnId, adj.getVrfId());
1916                 } else {
1917                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1918                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1919                 }
1920             }
1921         } catch (ReadFailedException e) {
1922             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1923                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1924         }
1925     }
1926
1927     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1928                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
1929                                     TypedWriteTransaction<Operational> writeOperTx) {
1930         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
1931         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1932         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1933             String vpnRd = vpn.getVrfId();
1934             if (vpnRd != null) {
1935                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
1936                         writeOperTx);
1937             }
1938         }
1939     }
1940
1941     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1942         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1943             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1944             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1945     }
1946
1947     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1948         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1949             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1950     }
1951
1952     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1953                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
1954         if (vpnInterface == null) {
1955             return;
1956         }
1957         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
1958         if (adjs == null) {
1959             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1960                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1961             return;
1962         }
1963         for (Adjacency adj : adjs) {
1964             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1965                 String primaryInterfaceIp = adj.getIpAddress();
1966                 String macAddress = adj.getMacAddress();
1967                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1968
1969                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
1970                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1971
1972                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1973                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1974                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1975                         routerInt, label, writeConfigTxn);
1976                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1977                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1978                         macAddress);
1979                 return;
1980             }
1981         }
1982         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1983                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1984                 primaryRd, vpnName);
1985     }
1986
1987     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1988             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
1989         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
1990         String rd = vpnUtil.getVpnRd(vpnName);
1991         if (adjs != null) {
1992             List<Adjacency> adjsList = adjs.getAdjacency();
1993             for (Adjacency adj : adjsList) {
1994                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1995                     String primaryInterfaceIp = adj.getIpAddress();
1996                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1997                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1998                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1999                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2000                     return;
2001                 }
2002             }
2003         } else {
2004             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2005                     vpnInterface.getName(), rd);
2006         }
2007     }
2008
2009     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2010         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
2011             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
2012                   vpnName, intefaceData.vpnInterface.getName());
2013             return;
2014         }
2015         final VpnInterfaceKey key = intefaceData.identifier
2016                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
2017         final String interfaceName = key.getName();
2018         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2019                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2020         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2021                   intefaceData.identifier, vpnName);
2022         return;
2023     }
2024
2025     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2026                                               VpnInterface vpnInterface, String vpnName) {
2027         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2028                .get(vpnName);
2029         if (vpnInterfaces == null) {
2030             vpnInterfaces = new ConcurrentLinkedQueue<>();
2031         }
2032         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2033         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2034         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2035                 vpnInterface.getName(), vpnName);
2036     }
2037
2038     public boolean isVpnInstanceReady(String vpnInstanceName) {
2039         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2040         if (vpnRd == null) {
2041             return false;
2042         }
2043         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2044
2045         return vpnInstanceOpDataEntry != null;
2046     }
2047
2048     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2049         synchronized (vpnInstanceName.intern()) {
2050             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2051                     unprocessedVpnInterfaces.get(vpnInstanceName);
2052             if (vpnInterfaces != null) {
2053                 while (!vpnInterfaces.isEmpty()) {
2054                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2055                     if (hasVpnInstanceCreatedSuccessfully) {
2056                         processSavedInterface(savedInterface, vpnInstanceName);
2057                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2058                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2059                     } else {
2060                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2061                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2062                     }
2063                 }
2064             } else {
2065                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2066             }
2067         }
2068     }
2069
2070     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2071             VpnInterface vpnInterface) {
2072         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2073             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2074                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2075             if (vpnInterfaces != null) {
2076                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2077                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2078                             + "unprocessed list", vpnInterface.getName(),
2079                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2080                 }
2081             } else {
2082                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2083                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2084             }
2085         }
2086     }
2087
2088     public void vpnInstanceIsReady(String vpnInstanceName) {
2089         processSavedInterfaces(vpnInstanceName, true);
2090     }
2091
2092     public void vpnInstanceFailed(String vpnInstanceName) {
2093         processSavedInterfaces(vpnInstanceName, false);
2094     }
2095
2096     private static class UnprocessedVpnInterfaceData {
2097         InstanceIdentifier<VpnInterface> identifier;
2098         VpnInterface vpnInterface;
2099
2100         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2101             this.identifier = identifier;
2102             this.vpnInterface = vpnInterface;
2103         }
2104
2105         @Override
2106         public int hashCode() {
2107             final int prime = 31;
2108             int result = 1;
2109             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2110             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2111             return result;
2112         }
2113
2114         @Override
2115         public boolean equals(Object obj) {
2116             if (this == obj) {
2117                 return true;
2118             }
2119             if (obj == null) {
2120                 return false;
2121             }
2122             if (getClass() != obj.getClass()) {
2123                 return false;
2124             }
2125             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2126             if (identifier == null) {
2127                 if (other.identifier != null) {
2128                     return false;
2129                 }
2130             } else if (!identifier.equals(other.identifier)) {
2131                 return false;
2132             }
2133             if (vpnInterface == null) {
2134                 if (other.vpnInterface != null) {
2135                     return false;
2136                 }
2137             } else if (!vpnInterface.equals(other.vpnInterface)) {
2138                 return false;
2139             }
2140             return true;
2141         }
2142     }
2143
2144     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2145         String primaryRd = vpnUtil.getVpnRd(vpnName);
2146         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2147         if (vpnInstanceOpData == null) {
2148             return;
2149         }
2150         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2151         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2152             return;
2153         }
2154         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2155         vpnToDpnLists.forEach(vpnToDpnList -> {
2156             if (vpnToDpnList.getVpnInterfaces() == null) {
2157                 return;
2158             }
2159             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2160                 try {
2161                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2162                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2163                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2164                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2165                     if (!vpnInterfaceOptional.isPresent()) {
2166                         return;
2167                     }
2168                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2169                             vpnInterface.getInterfaceName());
2170                     if (configVpnAdjacencies == null) {
2171                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2172                         return;
2173                     }
2174                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2175                             .augmentation(AdjacenciesOp.class).getAdjacency();
2176                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2177                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2178                     // in operational DS. These unprocessed adjacencies will be handled below.
2179                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2180                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2181                     configVpnAdjacencies.stream()
2182                         .filter(adjacency -> operationVpnAdjacencies.stream()
2183                                 .noneMatch(operationalAdjacency ->
2184                                         operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2185                         .forEach(adjacency -> {
2186                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2187                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2188                                 () -> {
2189                                     // TODO Deal with sequencing — the config tx must only submitted
2190                                     // if the oper tx goes in
2191                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2192                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2193                                         futures.add(
2194                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx ->
2195                                                 futures.add(
2196                                                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
2197                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2198                                                             primaryRd, adjacency, vpnInterfaceOptional.get()
2199                                                                 .getDpnId(), operTx, confTx)))));
2200                                         return futures;
2201                                     } else {
2202                                         return Collections.emptyList();
2203                                     }
2204                                 });
2205                         });
2206                 } catch (ReadFailedException e) {
2207                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2208                             vpnName, primaryRd);
2209                 }
2210             });
2211         });
2212     }
2213
2214     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2215         private final String interfaceName;
2216         private final boolean add;
2217         private final String txnDestination;
2218
2219         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2220             this.interfaceName = interfaceName;
2221             this.add = add;
2222             this.txnDestination = transactionDest;
2223         }
2224
2225         @Override
2226         public void onSuccess(Void voidObj) {
2227             if (add) {
2228                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2229                         interfaceName, txnDestination);
2230             } else {
2231                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2232             }
2233         }
2234
2235         @Override
2236         public void onFailure(Throwable throwable) {
2237             if (add) {
2238                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2239                         interfaceName, txnDestination, throwable);
2240             } else {
2241                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2242                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2243             }
2244         }
2245     }
2246 }