vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13
  14 import com.google.common.base.Optional;
  15 import com.google.common.base.Preconditions;
  16 import com.google.common.collect.Iterators;
  17 import com.google.common.util.concurrent.FutureCallback;
  18 import com.google.common.util.concurrent.Futures;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import com.google.common.util.concurrent.MoreExecutors;
  21 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  22 import java.math.BigInteger;
  23 import java.util.ArrayList;
  24 import java.util.Collections;
  25 import java.util.Iterator;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Objects;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ConcurrentLinkedQueue;
  31 import java.util.concurrent.ExecutionException;
  32 import java.util.function.Consumer;
  33 import java.util.function.Predicate;
  34 import java.util.stream.Collectors;
  35 import javax.annotation.PostConstruct;
  36 import javax.annotation.PreDestroy;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  41 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  42 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  45 import org.opendaylight.genius.infra.Datastore.Configuration;
  46 import org.opendaylight.genius.infra.Datastore.Operational;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  48 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  49 import org.opendaylight.genius.infra.TransactionAdapter;
  50 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  51 import org.opendaylight.genius.infra.TypedWriteTransaction;
  52 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  53 import org.opendaylight.genius.mdsalutil.NWUtil;
  54 import org.opendaylight.genius.mdsalutil.NwConstants;
  55 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  56 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  57 import org.opendaylight.infrautils.caches.CacheProvider;
  58 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  59 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  60 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  61 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  62 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  63 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  64 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  65 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  66 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  67 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  68 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  69 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  70 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  71 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  72 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  73 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  74 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  75 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  76 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 116 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 117 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 118 import org.slf4j.Logger;
 119 import org.slf4j.LoggerFactory;
 120
 121 @Singleton
 122 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 123
 124     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 125     private static final short DJC_MAX_RETRIES = 3;
 126
 127     private final DataBroker dataBroker;
 128     private final ManagedNewTransactionRunner txRunner;
 129     private final IBgpManager bgpManager;
 130     private final IFibManager fibManager;
 131     private final IMdsalApiManager mdsalManager;
 132     private final IdManagerService idManager;
 133     private final OdlInterfaceRpcService ifaceMgrRpcService;
 134     private final VpnFootprintService vpnFootprintService;
 135     private final IInterfaceManager interfaceManager;
 136     private final IVpnManager vpnManager;
 137     private final ArpResponderHandler arpResponderHandler;
 138     private final JobCoordinator jobCoordinator;
 139     private final VpnUtil vpnUtil;
 140
 141     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 142
 143     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 144             new ConcurrentHashMap<>();
 145
 146     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 147
 148     @Inject
 149     public VpnInterfaceManager(final DataBroker dataBroker,
 150                                final IBgpManager bgpManager,
 151                                final IdManagerService idManager,
 152                                final IMdsalApiManager mdsalManager,
 153                                final IFibManager fibManager,
 154                                final OdlInterfaceRpcService ifaceMgrRpcService,
 155                                final VpnFootprintService vpnFootprintService,
 156                                final IInterfaceManager interfaceManager,
 157                                final IVpnManager vpnManager,
 158                                final ArpResponderHandler arpResponderHandler,
 159                                final JobCoordinator jobCoordinator,
 160                                final CacheProvider cacheProvider,
 161                                final VpnUtil vpnUtil) {
 162         super(VpnInterface.class, VpnInterfaceManager.class);
 163
 164         this.dataBroker = dataBroker;
 165         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 166         this.bgpManager = bgpManager;
 167         this.idManager = idManager;
 168         this.mdsalManager = mdsalManager;
 169         this.fibManager = fibManager;
 170         this.ifaceMgrRpcService = ifaceMgrRpcService;
 171         this.vpnFootprintService = vpnFootprintService;
 172         this.interfaceManager = interfaceManager;
 173         this.vpnManager = vpnManager;
 174         this.arpResponderHandler = arpResponderHandler;
 175         this.jobCoordinator = jobCoordinator;
 176         this.vpnUtil = vpnUtil;
 177
 178         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 179                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 180                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 181     }
 182
 183     public Runnable isNotifyTaskQueued(String intfName) {
 184         return vpnIntfMap.remove(intfName);
 185     }
 186
 187     @PostConstruct
 188     public void start() {
 189         LOG.info("{} start", getClass().getSimpleName());
 190         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 191     }
 192
 193     @Override
 194     @PreDestroy
 195     public void close() {
 196         super.close();
 197         vpnInstanceOpDataEntryCache.close();
 198     }
 199
 200     @Override
 201     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 202         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 203     }
 204
 205     @Override
 206     protected VpnInterfaceManager getDataTreeChangeListener() {
 207         return VpnInterfaceManager.this;
 208     }
 209
 210     @Override
 211     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 212         LOG.info("add: intfName {} onto vpnName {}",
 213                  vpnInterface.getName(),
 214                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 215         addVpnInterface(identifier, vpnInterface, null, null);
 216     }
 217
 218     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 219                           final VpnInterface vpnInterface, String vpnName) {
 220         synchronized (vpnName.intern()) {
 221             if (isVpnInstanceReady(vpnName)) {
 222                 return true;
 223             }
 224             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 225             return false;
 226         }
 227     }
 228
 229     // TODO Clean up the exception handling
 230     @SuppressWarnings("checkstyle:IllegalCatch")
 231     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 232                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 233         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 234             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 235             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 236         }
 237     }
 238
 239     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 240                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 241         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 242         final String interfaceName = key.getName();
 243
 244         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 245             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 246                   vpnName, vpnInterface.getName());
 247             return;
 248         }
 249         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 250              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 251         List<Adjacency> copyOldAdjs = null;
 252         if (oldAdjs != null) {
 253             copyOldAdjs = new ArrayList<>();
 254             copyOldAdjs.addAll(oldAdjs);
 255         }
 256         List<Adjacency> copyNewAdjs = null;
 257         if (newAdjs != null) {
 258             copyNewAdjs = new ArrayList<>();
 259             copyNewAdjs.addAll(newAdjs);
 260         }
 261         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 262     }
 263
 264     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 265                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 266                     final List<Adjacency> newAdjs,
 267                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 268         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 269         final String interfaceName = key.getName();
 270         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 271         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 272             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 273             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 274             if (interfaceState != null) {
 275                 try {
 276                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 277                     final int ifIndex = interfaceState.getIfIndex();
 278                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 279                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 280                         // (the inventory tx goes in last)
 281                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 282                         ListenableFuture<Void> confFuture =
 283                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 284                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 285                                     operTx -> futures.add(
 286                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 287                                             LOG.info(
 288                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 289                                                     + " on dpn {}",
 290                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 291                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 292                                                 confTx, operTx, invTx, interfaceState, vpnName);
 293                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 294                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 295                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 296                                                 if (newAdjs != null) {
 297                                                     for (Adjacency adj : newAdjs) {
 298                                                         if (oldAdjs.contains(adj)) {
 299                                                             oldAdjs.remove(adj);
 300                                                         } else {
 301                                                             if (!isBgpVpnInternetVpn
 302                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 303                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 304                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx);
 305                                                             }
 306                                                         }
 307                                                     }
 308                                                 }
 309                                                 for (Adjacency adj : oldAdjs) {
 310                                                     if (!isBgpVpnInternetVpn
 311                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 312                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 313                                                             operTx, confTx);
 314                                                     }
 315                                                 }
 316                                             }
 317                                         })))));
 318                         futures.add(confFuture);
 319                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 320                             MoreExecutors.directExecutor());
 321                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 322                             + " processed successfully", interfaceName, vpnName, dpnId);
 323                         return futures;
 324                     });
 325                 } catch (NumberFormatException | IllegalStateException e) {
 326                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 327                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 328                         vpnName, e);
 329                     return;
 330                 }
 331             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 332                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 333                     () -> {
 334                         ListenableFuture<Void> future =
 335                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 336                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 337                                     confTx, vpnName);
 338                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 339                                     vpnName, vpnInterface.getDpnId());
 340                             });
 341                         ListenableFutures.addErrorLogging(future, LOG,
 342                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 343                         return Collections.singletonList(future);
 344                     });
 345             } else {
 346                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 347                     + " is not available", interfaceName, vpnName);
 348             }
 349         } else {
 350             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 351                     + " as vpn is pending delete", interfaceName, vpnName,
 352                 vpnInterface.getDpnId());
 353         }
 354     }
 355
 356     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 357     // see comments below.
 358     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 359     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 360             final int lportTag, boolean isInterfaceUp,
 361             TypedWriteTransaction<Configuration> writeConfigTxn,
 362             TypedWriteTransaction<Operational> writeOperTxn,
 363             TypedReadWriteTransaction<Configuration> writeInvTxn,
 364             Interface interfaceState,
 365             final String vpnName) throws ExecutionException, InterruptedException {
 366         final String interfaceName = vpnInterface.getName();
 367         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 368                 vpnName);
 369         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 370         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 371         if (!isInterfaceUp) {
 372             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 373                      interfaceName, dpId, vpnName);
 374             long vpnId = vpnUtil.getVpnId(vpnName);
 375             if (vpnId == VpnConstants.INVALID_ID) {
 376                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 377                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 378                         dpId);
 379                 return;
 380             }
 381
 382             boolean waitForVpnInterfaceOpRemoval = false;
 383             if (opVpnInterface != null) {
 384                 String opVpnName = opVpnInterface.getVpnInstanceName();
 385                 String primaryInterfaceIp = null;
 386                 if (opVpnName.equals(vpnName)) {
 387                     // Please check if the primary VRF Entry does not exist for VPNInterface
 388                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 389                     // back to back
 390                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 391                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 392                     if (adjs == null) {
 393                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 394                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 395                                 vpnName);
 396                         return;
 397                     }
 398                     for (Adjacency adj : adjs) {
 399                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 400                             primaryInterfaceIp = adj.getIpAddress();
 401                             break;
 402                         }
 403                     }
 404                     if (primaryInterfaceIp == null) {
 405                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 406                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 407                                 dpId, vpnName);
 408                         return;
 409                     }
 410                     // Get the rd of the vpn instance
 411                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 412                     if (vrf != null) {
 413                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 414                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 415                         return;
 416                     }
 417                     waitForVpnInterfaceOpRemoval = true;
 418                 } else {
 419                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 420                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 421                 }
 422             }
 423             if (!waitForVpnInterfaceOpRemoval) {
 424                 // Add the VPNInterface and quit
 425                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 426                         null/*ipAddressSourceValuePair*/,
 427                         true /* add */);
 428                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 429                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 430                 if (!isBgpVpnInternetVpn) {
 431                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 432                 }
 433                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 434                         dpId, vpnName);
 435                 if (interfaceManager.isExternalInterface(interfaceName)) {
 436                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 437                         NwConstants.ADD_FLOW);
 438                 }
 439                 return;
 440             }
 441
 442             // FIB didn't get a chance yet to clean up this VPNInterface
 443             // Let us give it a chance here !
 444             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 445                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 446             try {
 447                 Runnable notifyTask = new VpnNotifyTask();
 448                 synchronized (notifyTask) {
 449                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 450                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 451                     // published here while holding the lock on it so this path will hit the wait before notify can be
 452                     // invoked.
 453                     vpnIntfMap.put(interfaceName, notifyTask);
 454                     try {
 455                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 456                     } catch (InterruptedException e) {
 457                         // Ignored
 458                     }
 459                 }
 460             } finally {
 461                 vpnIntfMap.remove(interfaceName);
 462             }
 463
 464             if (opVpnInterface != null) {
 465                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 466                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 467                         dpId, vpnName);
 468                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 469                 return;
 470             }
 471             // VPNInterface got removed, proceed with Add
 472             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 473                     interfaceName, dpId, vpnName);
 474             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 475                     null/*ipAddressSourceValuePair*/,
 476                     true /* add */);
 477             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 478                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 479             if (!isBgpVpnInternetVpn) {
 480                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 481             }
 482             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 483                     + " FIB to clean up", interfaceName, dpId, vpnName);
 484             if (interfaceManager.isExternalInterface(interfaceName)) {
 485                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 486                                lportTag, NwConstants.ADD_FLOW);
 487             }
 488         } else {
 489             try {
 490                 // Interface is retained in the DPN, but its Link Up.
 491                 // Advertise prefixes again for this interface to BGP
 492                 InstanceIdentifier<VpnInterface> identifier =
 493                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 494                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 495                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 496                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 497                 // Perform similar operation as interface add event for extraroutes.
 498                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 499                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 500                         LogicalDatastoreType.CONFIGURATION, path);
 501                 if (!optAdjacencies.isPresent()) {
 502                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 503                     return;
 504                 }
 505                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 506                 for (Adjacency adjacency : adjacencies) {
 507                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 508                         continue;
 509                     }
 510                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 511                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 512                         continue;
 513                     }
 514                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 515                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn);
 516                 }
 517             } catch (ReadFailedException e) {
 518                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 519                         interfaceName, vpnName, primaryRd, dpId);
 520             }
 521         }
 522     }
 523
 524     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 525         int lportTag, int addOrRemove) {
 526         Uuid extNetworkId;
 527         try {
 528             // vpn instance of ext-net interface is the network-id
 529             extNetworkId = new Uuid(vpnName);
 530         } catch (IllegalArgumentException e) {
 531             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 532                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 533             return;
 534         }
 535
 536         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 537         if (routerIds == null || routerIds.isEmpty()) {
 538             LOG.info("processExternalVpnInterface: No router is associated with {}."
 539                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 540                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 541             return;
 542         }
 543
 544         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 545                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 546         for (Uuid routerId : routerIds) {
 547             String routerName = routerId.getValue();
 548             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 549             if (Objects.equals(primarySwitch, dpId)) {
 550                 Routers router = vpnUtil.getExternalRouter(routerName);
 551                 if (router != null) {
 552                     if (addOrRemove == NwConstants.ADD_FLOW) {
 553                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 554                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 555                                 dpId, interfaceName, lportTag);
 556                     } else {
 557                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 558                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 559                                 dpId, interfaceName, lportTag);
 560                     }
 561                 } else {
 562                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 563                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 564                             interfaceName, dpId, vpnName);
 565                 }
 566             }
 567         }
 568     }
 569
 570     // TODO Clean up the exception handling
 571     @SuppressWarnings("checkstyle:IllegalCatch")
 572     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 573                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 574                                                   String vpnName, String interfaceName) {
 575         if (rd == null) {
 576             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 577                   interfaceName, dpnId, vpnName);
 578             return;
 579         } else {
 580             if (rd.equals(vpnName)) {
 581                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 582                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 583                 return;
 584             }
 585         }
 586         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 587                 interfaceName, dpnId, vpnName, rd);
 588
 589         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 590         if (nextHopIp == null) {
 591             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 592                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 593                     rd, vpnName);
 594             return;
 595         }
 596
 597         try {
 598             //Read NextHops
 599             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 600             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 601                     LogicalDatastoreType.OPERATIONAL, path);
 602             if (adjacencies.isPresent()) {
 603                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 604                 if (!nextHops.isEmpty()) {
 605                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 606                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 607                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 608                     long l3vni = vpnInstanceOpData.getL3vni();
 609                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 610                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 611                     for (Adjacency nextHop : nextHops) {
 612                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 613                             continue;
 614                         }
 615                         String gatewayMac = null;
 616                         long label = 0;
 617                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 618                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 619                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 620                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 621                                     .get();
 622                         } else {
 623                             label = nextHop.getLabel();
 624                         }
 625                         try {
 626                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 627                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 628                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 629                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 630                                     gatewayMac);
 631                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 632                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 633                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 634                         } catch (Exception e) {
 635                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 636                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 637                                     interfaceName, dpnId, e);
 638                         }
 639                     }
 640                 }
 641             }
 642         } catch (ReadFailedException e) {
 643             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 644                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 645         }
 646     }
 647
 648     // TODO Clean up the exception handling
 649     @SuppressWarnings("checkstyle:IllegalCatch")
 650     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 651                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 652                         TypedWriteTransaction<Operational> writeOperTx) {
 653         //Read NextHops
 654         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 655         String rd = vpnUtil.getVpnRd(interfaceName);
 656         if (rd == null) {
 657             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 658                 interfaceName, vpnName);
 659             return;
 660         } else {
 661             if (rd.equals(vpnName)) {
 662                 LOG.info(
 663                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 664                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 665                 return;
 666             }
 667         }
 668         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 669                vpnName, rd);
 670         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 671         try {
 672             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 673                     path);
 674         } catch (ReadFailedException e) {
 675             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 676                     interfaceName, vpnName);
 677         }
 678         if (adjacencies.isPresent()) {
 679             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 680
 681             if (!nextHops.isEmpty()) {
 682                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 683                         nextHops, interfaceName, vpnName, rd);
 684                 for (Adjacency nextHop : nextHops) {
 685                     try {
 686                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 687                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 688                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 689                                     interfaceName, vpnName);
 690                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 691                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 692                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 693                                     interfaceName, vpnName);
 694                         } else {
 695                             // Perform similar operation as interface delete event for extraroutes.
 696                             String allocatedRd = nextHop.getVrfId();
 697                             for (String nh : nextHop.getNextHopIpList()) {
 698                                 deleteExtraRouteFromCurrentAndImportingVpns(
 699                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 700                                     writeOperTx);
 701                             }
 702                         }
 703                     } catch (Exception e) {
 704                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 705                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 706                     }
 707                 }
 708             }
 709         }
 710     }
 711
 712     @SuppressWarnings("checkstyle:IllegalCatch")
 713     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 714                                                   String primaryRd, String interfaceName, final long vpnId,
 715                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 716                                                   TypedWriteTransaction<Operational> writeOperTxn,
 717                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 718                                                   Interface interfaceState)
 719             throws ExecutionException, InterruptedException {
 720         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 721         // Read NextHops
 722         Optional<VpnInterface> vpnInteface = Optional.absent();
 723         try {
 724             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 725             LogicalDatastoreType.CONFIGURATION, identifier);
 726         } catch (ReadFailedException e) {
 727             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 728                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 729         }
 730         Uuid intfnetworkUuid = null;
 731         NetworkType networkType = null;
 732         Long segmentationId = Long.valueOf(-1);
 733         Adjacencies adjacencies = null;
 734         if (vpnInteface.isPresent()) {
 735             intfnetworkUuid = vpnInteface.get().getNetworkId();
 736             networkType = vpnInteface.get().getNetworkType();
 737             segmentationId = vpnInteface.get().getSegmentationId();
 738             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 739             if (adjacencies == null) {
 740                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 741                         null/*gwMac*/, writeOperTxn);
 742                 return;
 743             }
 744         }
 745         // Get the rd of the vpn instance
 746         String nextHopIp = null;
 747         try {
 748             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 749         } catch (Exception e) {
 750             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 751                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 752         }
 753         List<String> nhList = new ArrayList<>();
 754         if (nextHopIp != null) {
 755             nhList.add(nextHopIp);
 756             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 757                     interfaceName, dpnId, vpnName, nhList);
 758         }
 759         Optional<String> gwMac = Optional.absent();
 760         String vpnInterfaceSubnetGwMacAddress = null;
 761         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 762         Long l3vni = vpnInstanceOpData.getL3vni();
 763         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 764         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 765         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 766         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : Collections.emptyList();
 767         List<Adjacency> value = new ArrayList<>();
 768         for (Adjacency nextHop : nextHops) {
 769             String rd = primaryRd;
 770             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 771             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 772                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 773                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 774                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 775                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 776                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 777                 continue;
 778             }
 779             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 780                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 781                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 782                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 783                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 784                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 785
 786                 Prefixes prefixes = (intfnetworkUuid != null)
 787                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 788                             segmentationId, prefixCue) :
 789                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 790                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 791                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 792                 final Uuid subnetId = nextHop.getSubnetId();
 793
 794                 String gatewayIp = nextHop.getSubnetGatewayIp();
 795                 if (gatewayIp == null) {
 796                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 797                     if (gatewayIpOptional.isPresent()) {
 798                         gatewayIp = gatewayIpOptional.get();
 799                     }
 800                 }
 801
 802                 if (gatewayIp != null) {
 803                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 804                     if (gwMac.isPresent()) {
 805                         // A valid mac-address is available for this subnet-gateway-ip
 806                         // Use this for programming ARP_RESPONDER table here.  And save this
 807                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 808                         // to populate L3_GW_MAC_TABLE there.
 809                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 810                                 gatewayIp, gwMac.get());
 811                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 812                     } else {
 813                         // A valid mac-address is not available for this subnet-gateway-ip
 814                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 815                         // Save this connected-mac-address as gateway-mac-address for the
 816                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 817                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 818                         if (gwMac.isPresent()) {
 819                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 820                                     NwConstants.ADD_FLOW, gwMac.get());
 821                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 822                                     gatewayIp, gwMac.get());
 823                         } else {
 824                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 825                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 826                                 + "gwIp {}",
 827                                 subnetId, interfaceName, vpnName, gatewayIp);
 828                         }
 829                     }
 830                 } else {
 831                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 832                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 833                         subnetId, interfaceName, vpnName);
 834                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 835                 }
 836                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 837                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 838             } else {
 839                 //Extra route adjacency
 840                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 841                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 842                 synchronized (vpnPrefixKey.intern()) {
 843                     java.util.Optional<String> rdToAllocate = vpnUtil
 844                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 845                                     nextHop.getNextHopIpList().get(0), dpnId);
 846                     if (rdToAllocate.isPresent()) {
 847                         rd = rdToAllocate.get();
 848                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 849                             rd, prefix);
 850                     } else {
 851                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 852                         continue;
 853                     }
 854                 }
 855                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 856                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 857                         interfaceName, dpnId);
 858             }
 859             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 860             // can be different from the gateway-mac-address within the VRFEntry as the
 861             // gateway-mac-address is a superset.
 862             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 863                     : RouteOrigin.STATIC;
 864             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 865                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 866                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 867             Adjacency operationalAdjacency = null;
 868             try {
 869                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 870             } catch (NullPointerException e) {
 871                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 872                     input, e.getMessage());
 873                 return;
 874             }
 875             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 876                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 877                         vpnName, l3vni, origin,
 878                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 879             }
 880             value.add(operationalAdjacency);
 881         }
 882
 883         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 884         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 885                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 886
 887         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 888                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 889                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 890
 891         for (Adjacency nextHop : aug.getAdjacency()) {
 892             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 893             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 894                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 895                         : RouteOrigin.STATIC;
 896                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 897                 registeredPopulator.populateFib(input, writeConfigTxn);
 898             }
 899         }
 900     }
 901
 902     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 903                                               long lportTag, String gwMac,
 904                                               TypedWriteTransaction<Operational> writeOperTxn) {
 905         VpnInterfaceOpDataEntry opInterface =
 906               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 907         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 908             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 909         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 910         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 911                 interfaceName, dpnId, vpnName);
 912     }
 913
 914     // TODO Clean up the exception handling
 915     @SuppressWarnings("checkstyle:IllegalCatch")
 916     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 917                                            StateTunnelList stateTunnelList,
 918                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 919                                            TypedWriteTransaction<Operational> writeOperTxn) {
 920
 921         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 922         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 923         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 924         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 925         if (adjList.isEmpty()) {
 926             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 927                     vpnInterface, srcDpnId);
 928             return;
 929         }
 930         String prefix = null;
 931         long label = 0;
 932         List<Adjacency> value = new ArrayList<>();
 933         boolean isNextHopAddReqd = false;
 934         String vpnName = vpnInterface.getVpnInstanceName();
 935         long vpnId = vpnUtil.getVpnId(vpnName);
 936         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 937         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 938                 vpnInterface.getName(), vpnInterface.getDpnId(),
 939                 vpnInterface.getVpnInstanceName(), adjList);
 940         for (Adjacency adj : adjList) {
 941             String rd = adj.getVrfId();
 942             rd = rd != null ? rd : vpnName;
 943             prefix = adj.getIpAddress();
 944             label = adj.getLabel();
 945             List<String> nhList = Collections.singletonList(srcTepIp);
 946             List<String> nextHopList = adj.getNextHopIpList();
 947             // If TEP is added , update the nexthop of primary adjacency.
 948             // Secondary adj nexthop is already pointing to primary adj IP address.
 949             if (nextHopList != null && !nextHopList.isEmpty()) {
 950                 /* everything right already */
 951             } else {
 952                 isNextHopAddReqd = true;
 953             }
 954
 955             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 956                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 957             } else {
 958                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 959                 if (!vrfEntryOptional.isPresent()) {
 960                     continue;
 961                 }
 962                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 963                 if (!nhList.contains(srcTepIp)) {
 964                     nhList.add(srcTepIp);
 965                     isNextHopAddReqd = true;
 966                 }
 967                 value.add(adj);
 968             }
 969
 970             if (isNextHopAddReqd) {
 971                 updateLabelMapper(label, nhList);
 972                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 973                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 974                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 975                 // Update the VRF entry with nextHop
 976                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 977                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 978
 979                 //Get the list of VPN's importing this route(prefix) .
 980                 // Then update the VRF entry with nhList
 981                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 982                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 983                     String vpnRd = vpn.getVrfId();
 984                     if (vpnRd != null) {
 985                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 986                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 987                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 988                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 989                             vpnInterface.getName(), srcDpnId, vpnName,
 990                             vpn.getVpnInstanceName(), vpnRd);
 991                     }
 992                 }
 993                 // Advertise the prefix to BGP only for external vpn
 994                 // since there is a nexthop change.
 995                 try {
 996                     if (!rd.equalsIgnoreCase(vpnName)) {
 997                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 998                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 999                                 null /*gatewayMacAddress*/);
1000                     }
1001                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1002                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1003                             srcDpnId, vpnName);
1004                 } catch (Exception ex) {
1005                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1006                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1007                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1008                 }
1009             }
1010         }
1011         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1012         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1013                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1014                 .addAugmentation(AdjacenciesOp.class, aug).build();
1015         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1016                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1017         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1018         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1019                 vpnInterface.getName(), srcDpnId, vpnName);
1020
1021     }
1022
1023     // TODO Clean up the exception handling
1024     @SuppressWarnings("checkstyle:IllegalCatch")
1025     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1026                                               StateTunnelList stateTunnelList,
1027                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1028                                               TypedWriteTransaction<Operational> writeOperTxn) {
1029
1030         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1031         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1032         String prefix = null;
1033         long label = 0;
1034         boolean isNextHopRemoveReqd = false;
1035         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1036         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1037         String vpnName = vpnInterface.getVpnInstanceName();
1038         long vpnId = vpnUtil.getVpnId(vpnName);
1039         String primaryRd = vpnUtil.getVpnRd(vpnName);
1040         if (adjList != null) {
1041             List<Adjacency> value = new ArrayList<>();
1042             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1043                     vpnInterface.getName(), vpnInterface.getDpnId(),
1044                     vpnInterface.getVpnInstanceName(), adjList);
1045             for (Adjacency adj : adjList) {
1046                 List<String> nhList = new ArrayList<>();
1047                 String rd = adj.getVrfId();
1048                 rd = rd != null ? rd : vpnName;
1049                 prefix = adj.getIpAddress();
1050                 List<String> nextHopList = adj.getNextHopIpList();
1051                 label = adj.getLabel();
1052                 if (nextHopList != null && !nextHopList.isEmpty()) {
1053                     isNextHopRemoveReqd = true;
1054                 }
1055                 // If TEP is deleted , remove the nexthop from primary adjacency.
1056                 // Secondary adj nexthop will continue to point to primary adj IP address.
1057                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1058                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1059                 } else {
1060                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1061                     if (!vrfEntryOptional.isPresent()) {
1062                         continue;
1063                     }
1064                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1065                     if (nhList.contains(srcTepIp)) {
1066                         nhList.remove(srcTepIp);
1067                         isNextHopRemoveReqd = true;
1068                     }
1069                     value.add(adj);
1070                 }
1071
1072                 if (isNextHopRemoveReqd) {
1073                     updateLabelMapper(label, nhList);
1074                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1075                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1076                             prefix, nhList, vpnName,
1077                             vpnId, rd, vpnInterface.getName());
1078                     // Update the VRF entry with removed nextHop
1079                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1080                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1081
1082                     //Get the list of VPN's importing this route(prefix) .
1083                     // Then update the VRF entry with nhList
1084                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1085                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1086                         String vpnRd = vpn.getVrfId();
1087                         if (vpnRd != null) {
1088                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1089                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1090                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1091                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1092                                     nhList, label, vpnInterface.getName(), srcDpnId,
1093                                     vpnName,
1094                                     vpn.getVpnInstanceName(), vpnRd);
1095                         }
1096                     }
1097
1098                     // Withdraw prefix from BGP only for external vpn.
1099                     try {
1100                         if (!rd.equalsIgnoreCase(vpnName)) {
1101                             bgpManager.withdrawPrefix(rd, prefix);
1102                         }
1103                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1104                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1105                                 vpnInterface.getName(), srcDpnId,
1106                                 vpnName);
1107                     } catch (Exception ex) {
1108                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1109                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1110                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1111                     }
1112                 }
1113             }
1114             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1115             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1116                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1117                     .addAugmentation(AdjacenciesOp.class, aug).build();
1118             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1119                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1120             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1121             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1122                          vpnInterface.getName(), srcDpnId, vpnName);
1123         }
1124     }
1125
1126     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1127         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1128
1129         String vpnRd = vpnUtil.getVpnRd(vpnName);
1130         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1131         if (vpnInstanceOpDataEntry == null) {
1132             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1133                     + " to check for vpns exporting the routes", vpnName);
1134             return vpnsToExportRoute;
1135         }
1136
1137         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1138             if (input.getVpnInstanceName() == null) {
1139                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1140                         input.getVrfId());
1141                 return false;
1142             }
1143             return !input.getVpnInstanceName().equals(vpnName);
1144         };
1145
1146         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1147             Iterable<String> commonRTs =
1148                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1149                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1150             return Iterators.size(commonRTs.iterator()) > 0;
1151         };
1152
1153         vpnsToExportRoute =
1154                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1155                         Collectors.toList());
1156         return vpnsToExportRoute;
1157     }
1158
1159     // TODO Clean up the exception handling
1160     @SuppressWarnings("checkstyle:IllegalCatch")
1161     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1162         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1163         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1164             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1165             if (vrfEntries != null) {
1166                 ListenableFutures.addErrorLogging(
1167                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1168                         for (VrfEntry vrfEntry : vrfEntries) {
1169                             try {
1170                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1171                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1172                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1173                                             + " is not a controller managed non intervpn link route. Ignoring.",
1174                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1175                                     continue;
1176                                 }
1177                                 String prefix = vrfEntry.getDestPrefix();
1178                                 String gwMac = vrfEntry.getGatewayMacAddress();
1179                                 vrfEntry.getRoutePaths().forEach(routePath -> {
1180                                     String nh = routePath.getNexthopAddress();
1181                                     int label = routePath.getLabel().intValue();
1182                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1183                                         vrfEntry.getOrigin()))) {
1184                                         LOG.info(
1185                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1186                                                 + " nexthop {} label {} to vpn {} vpnRd {}",
1187                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1188                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1189                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1190                                             0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1191                                             confTx);
1192                                     } else {
1193                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1194                                                 + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1195                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1196                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1197                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1198                                             confTx);
1199                                     }
1200                                 });
1201                             } catch (RuntimeException e) {
1202                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1203                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1204                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1205                             }
1206                         }
1207                     }), LOG, "Error handing VPN exporting routes");
1208             } else {
1209                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1210                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1211             }
1212         }
1213     }
1214
1215     @Override
1216     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1217         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1218         final String interfaceName = key.getName();
1219         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1220             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1221             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1222         }
1223     }
1224
1225     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1226                                 final VpnInterface vpnInterface, final String vpnName,
1227                                 final String interfaceName) {
1228         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1229             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1230                 ListenableFuture<Void> future =
1231                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1232                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1233                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1234                     });
1235                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1236                         vpnInterface.getName(), vpnName);
1237                 return Collections.singletonList(future);
1238             }, DJC_MAX_RETRIES);
1239         } else {
1240             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1241             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1242         }
1243     }
1244
1245     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1246     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1247                                            final VpnInterface vpnInterface, final String vpnName,
1248                                            final String interfaceName, final Interface interfaceState) {
1249         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1250                 vpnName, vpnInterface.getDpnId());
1251         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1252         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1253             () -> {
1254                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1255                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1256                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1257                         writeOperTxn -> futures.add(
1258                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1259                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1260                                         interfaceName, vpnName);
1261                                 BigInteger dpId = BigInteger.ZERO;
1262                                 int ifIndex = 0;
1263                                 String gwMacAddress = null;
1264                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1265                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1266                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1267                                 try {
1268                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1269                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1270                                 } catch (ReadFailedException e) {
1271                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1272                                             interfaceName, vpnName);
1273                                     return;
1274                                 }
1275                                 if (interfaceState != null) {
1276                                     try {
1277                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1278                                     } catch (NumberFormatException | IllegalStateException e) {
1279                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1280                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1281                                                         + " from vpn interface op data store. ", interfaceName,
1282                                                 vpnInterface.getDpnId(), vpnName, e);
1283                                         dpId = BigInteger.ZERO;
1284                                     }
1285                                     ifIndex = interfaceState.getIfIndex();
1286                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1287                                 } else {
1288                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1289                                             + " from vpn interface op.", interfaceName);
1290                                     if (optVpnInterface.isPresent()) {
1291                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1292                                         dpId = vpnOpInterface.getDpnId();
1293                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1294                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1295                                     } else {
1296                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1297                                                 + " as interfaceState and vpn interface op is not"
1298                                                 + " available", interfaceName, vpnName);
1299                                         return;
1300                                     }
1301                                 }
1302                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1303                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1304                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1305                                 LOG.info(
1306                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1307                                                 + "successfully",
1308                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1309                             })))));
1310                 futures.add(configFuture);
1311                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1312                 return futures;
1313             }, DJC_MAX_RETRIES);
1314     }
1315
1316     protected void processVpnInterfaceDown(BigInteger dpId,
1317                                            String interfaceName,
1318                                            int lportTag,
1319                                            String gwMac,
1320                                            VpnInterfaceOpDataEntry vpnOpInterface,
1321                                            boolean isInterfaceStateDown,
1322                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1323                                            TypedWriteTransaction<Operational> writeOperTxn,
1324                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1325             throws ExecutionException, InterruptedException {
1326         if (vpnOpInterface == null) {
1327             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1328                     + " as it is not available in operational data store", interfaceName, dpId);
1329             return;
1330         }
1331         final String vpnName = vpnOpInterface.getVpnInstanceName();
1332         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1333                                                     interfaceName, vpnName);
1334         if (!isInterfaceStateDown) {
1335             final long vpnId = vpnUtil.getVpnId(vpnName);
1336             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1337             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1338             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1339                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1340             if (interfaceManager.isExternalInterface(interfaceName)) {
1341                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1342                     NwConstants.DEL_FLOW);
1343             }
1344             if (!isBgpVpnInternetVpn) {
1345                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1346             }
1347             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1348                     + " successful", interfaceName, dpId, vpnName);
1349         } else {
1350             // Interface is retained in the DPN, but its Link Down.
1351             // Only withdraw the prefixes for this interface from BGP
1352             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1353         }
1354     }
1355
1356     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1357                                           final String vpnName, final long vpnId, String gwMac,
1358                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1359                                           TypedWriteTransaction<Operational> writeOperTxn,
1360                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1361             throws ExecutionException, InterruptedException {
1362         //Read NextHops
1363         try {
1364             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1365                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1366             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1367             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1368                     LogicalDatastoreType.OPERATIONAL, path);
1369             String primaryRd = vpnUtil.getVpnRd(vpnName);
1370             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1371                     interfaceName, dpnId, vpnName, primaryRd);
1372             if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1373                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1374                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1375                         interfaceName, dpnId, vpnName, nextHops);
1376                 for (Adjacency nextHop : nextHops) {
1377                     if (nextHop.isPhysNetworkFunc()) {
1378                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1379                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1380                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1381                                 null/*writeCfgTxn*/);
1382                     } else {
1383                         String rd = nextHop.getVrfId();
1384                         List<String> nhList;
1385                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1386                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1387                         } else {
1388                             // This is a primary adjacency
1389                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1390                                     : Collections.emptyList();
1391                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1392                                     interfaceName, writeInvTxn);
1393                         }
1394                         if (!nhList.isEmpty()) {
1395                             if (rd.equals(vpnName)) {
1396                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1397                                 //remove from FIB directly
1398                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1399                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1400                             } else {
1401                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1402                                         interfaceName, writeConfigTxn, writeOperTxn);
1403                             }
1404                         } else {
1405                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1406                                             + " interface {}", nextHop.getIpAddress(), rd,
1407                                     nextHop.getAdjacencyType().toString(), interfaceName);
1408                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1409                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1410                         }
1411                     }
1412                     String ip = nextHop.getIpAddress().split("/")[0];
1413                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1414                     if (vpnVipToPort != null) {
1415                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1416                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1417                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1418                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1419                     }
1420                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1421                     if (vpnPortipToPort != null) {
1422                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1423                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1424                                  + "Interface {} ip {} on dpn {} for vpn {}",
1425                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1426                     }
1427                 }
1428             } else {
1429                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1430                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1431                         + " Removing it.", interfaceName, vpnName, dpnId);
1432                 writeOperTxn.delete(identifier);
1433             }
1434         } catch (ReadFailedException e) {
1435             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1436                     interfaceName, dpnId, vpnName);
1437         }
1438     }
1439
1440     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1441                                                             String interfaceName, BigInteger dpnId,
1442                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1443                                                             TypedWriteTransaction<Operational> writeOperTx) {
1444         return (nh) -> {
1445             String primaryRd = vpnUtil.getVpnRd(vpnName);
1446             String prefix = nextHop.getIpAddress();
1447             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1448             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1449                     nextHop, vpnName, interfaceName, dpnId);
1450             synchronized (vpnNamePrefixKey.intern()) {
1451                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1452                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1453                     //If extra-route is present behind at least one VM, then do not remove or update
1454                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1455                     //prefixtointerface DS
1456                     return;
1457                 }
1458                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1459                         writeConfigTxn);
1460             }
1461             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1462                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1463                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1464         };
1465     }
1466
1467     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1468                                            BigInteger dpnId, String rd, String interfaceName,
1469                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1470                                            TypedWriteTransaction<Operational> writeOperTx) {
1471         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1472                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1473         nhList.forEach((nh) -> {
1474             //IRT: remove routes from other vpns importing it
1475             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1476                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1477             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1478                 String vpnRd = vpn.getVrfId();
1479                 if (vpnRd != null) {
1480                     fibManager.removeOrUpdateFibEntry(vpnRd,
1481                             nextHop.getIpAddress(), nh, writeConfigTxn);
1482                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1483                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1484                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1485                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1486                 }
1487             }
1488         });
1489     }
1490
1491     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1492                                                  int lportTag, String gwMac, String interfaceName,
1493                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1494             throws ExecutionException, InterruptedException {
1495         final Uuid subnetId = nextHop.getSubnetId();
1496         if (nextHop.getSubnetGatewayMacAddress() == null) {
1497             // A valid mac-address was not available for this subnet-gateway-ip
1498             // So a connected-mac-address was used for this subnet and we need
1499             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1500             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1501         }
1502         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1503                 subnetId);
1504     }
1505
1506     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1507                                                   String interfaceName) {
1508         // This is either an extra-route (or) a learned IP via subnet-route
1509         List<String> nhList = null;
1510         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1511         if (nextHopIp == null || nextHopIp.isEmpty()) {
1512             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1513                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1514                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1515                     vpnName);
1516             nhList = Collections.emptyList();
1517         } else {
1518             nhList = Collections.singletonList(nextHopIp);
1519         }
1520         return nhList;
1521     }
1522
1523     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1524         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1525         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1526         // else use connected interface
1527         if (gwPort != null && gwPort.isSubnetIp()) {
1528             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1529                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1530             return Optional.of(gwPort.getMacAddress());
1531         }
1532         return Optional.absent();
1533     }
1534
1535     @Override
1536     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1537         final VpnInterface update) {
1538         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1539                 update.getDpnId(), original.getVpnInstanceNames(),
1540                 update.getVpnInstanceNames());
1541         final String vpnInterfaceName = update.getName();
1542         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1543         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1544         //handles switching between <internal VPN - external VPN>
1545         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1546             List<ListenableFuture<Void>> futures = new ArrayList<>();
1547             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1548                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1549                                 + "to newVpn(s) {}",
1550                         original.getName(), dpnId,
1551                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1552                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1553                 return Collections.emptyList();
1554             }
1555             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1556             return futures;
1557         });
1558     }
1559
1560     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1561                                                            VpnInterface original, VpnInterface update,
1562                                                            List<ListenableFuture<Void>> futures) {
1563         boolean isVpnInstanceUpdate = Boolean.FALSE;
1564         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1565         final String interfaceName = key.getName();
1566         List<String> oldVpnList = vpnUtil.getVpnListForVpnInterface(original);
1567         List<String> oldVpnListCopy = new ArrayList<>();
1568         oldVpnListCopy.addAll(oldVpnList);
1569         List<String> newVpnList = vpnUtil.getVpnListForVpnInterface(update);
1570
1571         oldVpnList.removeAll(newVpnList);
1572         newVpnList.removeAll(oldVpnListCopy);
1573         //This block will execute only on if there is a change in the VPN Instance.
1574         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1575             /*
1576              * Internet BGP-VPN Instance update with single router:
1577              * ====================================================
1578              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1579              *     1st VPN Instance : router VPN or external BGP-VPN.
1580              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1581              *
1582              * VPN Instance UPDATE:
1583              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1584              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1585              *
1586              * External BGP-VPN Instance update with single router:
1587              * ====================================================
1588              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1589              *
1590              * Updated VPN Instance will be always either internal router VPN to
1591              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1592              *
1593              * VPN Instance UPDATE:
1594              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1595              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1596              *
1597              * TODO Two router VPN instance update use case will be addressed in separate patch
1598              */
1599             isVpnInstanceUpdate = Boolean.TRUE;
1600             updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1601                     oldVpnListCopy, futures);
1602         }
1603         return isVpnInstanceUpdate;
1604     }
1605
1606     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1607                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1608                                          List<String> newVpnList, List<String> oldVpnListCopy,
1609                                          List<ListenableFuture<Void>> futures) {
1610         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1611         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1612                 ? origAdjs.getAdjacency() : new ArrayList<>();
1613         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1614         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1615                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1616
1617         boolean isOldVpnRemoveCallExecuted = false;
1618         for (String oldVpnName : oldVpnList) {
1619             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1620                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1621             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1622             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1623                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1624                     newVpnList);
1625             isOldVpnRemoveCallExecuted = true;
1626         }
1627         //Wait for previous interface bindings to be removed
1628         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1629             try {
1630                 Thread.sleep(2000);
1631             } catch (InterruptedException e) {
1632                 //Ignore
1633             }
1634         }
1635         for (String newVpnName : newVpnList) {
1636             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1637             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1638                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1639                         + "onto vpnName {} ", interfaceName, newVpnName);
1640                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1641                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1642                                 + "from oldVpn(s) {} to newVpn {} ",
1643                         interfaceName, oldVpnListCopy, newVpnName);
1644                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1645                  * Use Case:
1646                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1647                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1648                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1649                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1650                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1651                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1652                  *     router VPN as well as Internet BGP-VPN.
1653                  *
1654                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1655                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1656                  *                                       update in existing router VPN instance.
1657                  */
1658                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1659                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1660                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1661                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1662                 }
1663             }
1664         }
1665     }
1666
1667     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1668                                                                     String vpnInterfaceName,
1669                                                                     List<ListenableFuture<Void>> futures) {
1670         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1671         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1672                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1673         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1674         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1675                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1676
1677         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1678         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1679             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1680             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1681             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1682             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1683             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1684                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1685                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1686                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
1687                         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1688                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1689                         LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1690                                 update.getName(), newVpnName);
1691                         //handle both addition and removal of adjacencies
1692                         //currently, new adjacency may be an extra route
1693                         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1694                         if (!oldAdjs.equals(newAdjs)) {
1695                             for (Adjacency adj : copyNewAdjs) {
1696                                 if (copyOldAdjs.contains(adj)) {
1697                                     copyOldAdjs.remove(adj);
1698                                 } else {
1699                                     // add new adjacency
1700                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1701                                         addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1702                                                 dpnId, operTx, confTx, confTx);
1703                                     }
1704                                     LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1705                                                     + " vpn interface {} on vpn {} dpnId {}",
1706                                             adj.getIpAddress(), adj.getNextHopIpList(),
1707                                             adj.getLabel(), adj.getSubnetId(), update.getName(),
1708                                             newVpnName, dpnId);
1709                                 }
1710                             }
1711                             for (Adjacency adj : copyOldAdjs) {
1712                                 if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1713                                     if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1714                                             && !adj.isPhysNetworkFunc()) {
1715                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1716                                                 operTx, confTx);
1717                                         //remove FIB entry
1718                                         String vpnRd = vpnUtil.getVpnRd(newVpnName);
1719                                         LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1720                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1721                                         //remove BGP entry
1722                                         fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1723                                         if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1724                                             bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1725                                         }
1726                                     } else {
1727                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1728                                                 operTx, confTx);
1729                                     }
1730                                 }
1731                                 LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1732                                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1733                                                 .getNextHopIpList(),
1734                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1735                             }
1736                         }
1737                     }));
1738                 }));
1739                 for (ListenableFuture<Void> future : futures) {
1740                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1741                             update.getName(), update.getVpnInstanceNames());
1742                 }
1743             } else {
1744                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1745                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1746             }
1747         }
1748         return futures;
1749     }
1750
1751     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1752         try {
1753             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1754             synchronized (label.toString().intern()) {
1755                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1756                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1757                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1758                         LogicalDatastoreType.OPERATIONAL, lriIid);
1759                 if (opResult.isPresent()) {
1760                     LabelRouteInfo labelRouteInfo =
1761                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1762                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1763                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1764                 }
1765             }
1766             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1767                     nextHopIpList);
1768         } catch (ReadFailedException e) {
1769             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1770                     nextHopIpList);
1771         } catch (TransactionCommitFailedException e) {
1772             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1773                     nextHopIpList);
1774         }
1775     }
1776
1777     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1778         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1779
1780         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1781         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1782                 .addAugmentation(SubnetRoute.class, route).build();
1783         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1784         InstanceIdentifierBuilder<VrfTables> idBuilder =
1785             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1786         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1787         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1788         if (writeConfigTxn != null) {
1789             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1790         } else {
1791             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1792         }
1793         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1794                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1795     }
1796
1797     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1798                                            Adjacency adj, BigInteger dpnId,
1799                                            TypedWriteTransaction<Operational> writeOperTxn,
1800                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1801                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1802             throws ExecutionException, InterruptedException {
1803         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1804         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1805         try {
1806             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1807                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1808             if (optVpnInterface.isPresent()) {
1809                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1810                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1811                 String vpnName = currVpnIntf.getVpnInstanceName();
1812                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1813                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1814                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1815                         LogicalDatastoreType.OPERATIONAL, adjPath);
1816                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1817                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1818                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1819                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1820                 List<Adjacency> adjacencies;
1821                 if (optAdjacencies.isPresent()) {
1822                     adjacencies = optAdjacencies.get().getAdjacency();
1823                 } else {
1824                     // This code will be hit in case of first PNF adjacency
1825                     adjacencies = new ArrayList<>();
1826                 }
1827                 long vpnId = vpnUtil.getVpnId(vpnName);
1828                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1829                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1830                 Adjacency operationalAdjacency = null;
1831                 //Handling dual stack neutron port primary adjacency
1832                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1833                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1834                             currVpnIntf.getName(), vpnName);
1835                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1836                             currVpnIntf.getName());
1837                     if (interfaceState != null) {
1838                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1839                                 currVpnIntf.getName(),
1840                                 vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
1841                     }
1842                 }
1843                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1844                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1845                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1846                             : RouteOrigin.STATIC;
1847                     String nh = adj.getNextHopIpList().get(0);
1848                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1849                     synchronized (vpnPrefixKey.intern()) {
1850                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1851                                 vpnId, null, prefix, vpnName, nh, dpnId);
1852                         if (rdToAllocate.isPresent()) {
1853                             input.setRd(rdToAllocate.get());
1854                             operationalAdjacency = populator.createOperationalAdjacency(input);
1855                             int label = operationalAdjacency.getLabel().intValue();
1856                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1857                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1858                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1859                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1860                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1861                                     vpnName, label, currVpnIntf.getName(), dpnId);
1862                         } else {
1863                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1864                                     vpnName, prefix);
1865                             return;
1866                         }
1867                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1868                         // Keeping the MPLS check for now.
1869                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1870                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1871                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1872                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1873                             vpnsToImportRoute.forEach(vpn -> {
1874                                 if (vpn.getVrfId() != null) {
1875                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1876                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1877                                             .ifPresent(
1878                                                 rds -> vpnManager.addExtraRoute(
1879                                                         vpnUtil.getVpnName(vpn.getVpnId()),
1880                                                         adj.getIpAddress(), nh, rds,
1881                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1882                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1883                                                         opAdjacency, encapType, writeConfigTxn));
1884                                 }
1885                             });
1886                         }
1887                     }
1888                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1889                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1890                             currVpnIntf.getName(), vpnName);
1891
1892                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1893                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1894                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1895                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1896                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1897                             Prefixes.PrefixCue.PhysNetFunc);
1898                     if (vpnIntefaceConfig.isPresent()) {
1899                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1900                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1901                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1902                     }
1903
1904                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1905
1906                     writeOperTxn.merge(
1907                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1908                                     prefix), pnfPrefix, true);
1909
1910                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1911                             adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1912                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1913
1914                     input.setRd(adj.getVrfId());
1915                 }
1916                 if (operationalAdjacency == null) {
1917                     operationalAdjacency = populator.createOperationalAdjacency(input);
1918                 }
1919                 adjacencies.add(operationalAdjacency);
1920                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1921                 VpnInterfaceOpDataEntry newVpnIntf =
1922                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1923                                 aug, dpnId, currVpnIntf.getLportTag(),
1924                                 currVpnIntf.getGatewayMacAddress());
1925
1926                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
1927             }
1928         } catch (ReadFailedException e) {
1929             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1930                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1931         }
1932     }
1933
1934     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1935         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
1936         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1937     }
1938
1939     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1940                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
1941                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
1942         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1943         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1944         try {
1945             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1946                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1947             if (optVpnInterface.isPresent()) {
1948                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1949                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1950                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1951                         LogicalDatastoreType.OPERATIONAL, path);
1952                 if (optAdjacencies.isPresent()) {
1953                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1954
1955                     if (!adjacencies.isEmpty()) {
1956                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1957                         Iterator<Adjacency> adjIt = adjacencies.iterator();
1958                         while (adjIt.hasNext()) {
1959                             Adjacency adjElem = adjIt.next();
1960                             if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1961                                 String rd = adjElem.getVrfId();
1962                                 if (adj.getNextHopIpList() != null) {
1963                                     for (String nh : adj.getNextHopIpList()) {
1964                                         deleteExtraRouteFromCurrentAndImportingVpns(
1965                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1966                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
1967                                     }
1968                                 } else if (adj.isPhysNetworkFunc()) {
1969                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1970                                             adj.getIpAddress(), adj.getSubnetId());
1971                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1972                                             writeConfigTxn);
1973                                 }
1974                                 break;
1975                             }
1976
1977                         }
1978                     }
1979                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1980                             dpnId, adj.getVrfId());
1981                 } else {
1982                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1983                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1984                 }
1985             }
1986         } catch (ReadFailedException e) {
1987             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1988                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1989         }
1990     }
1991
1992     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1993                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
1994                                     TypedWriteTransaction<Operational> writeOperTx) {
1995         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
1996         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1997         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1998             String vpnRd = vpn.getVrfId();
1999             if (vpnRd != null) {
2000                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2001                         writeOperTx);
2002             }
2003         }
2004     }
2005
2006     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2007         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2008             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2009             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2010     }
2011
2012     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2013         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2014             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2015     }
2016
2017     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2018                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2019         if (vpnInterface == null) {
2020             return;
2021         }
2022         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2023         if (adjs == null) {
2024             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2025                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2026             return;
2027         }
2028         for (Adjacency adj : adjs) {
2029             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2030                 String primaryInterfaceIp = adj.getIpAddress();
2031                 String macAddress = adj.getMacAddress();
2032                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2033
2034                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2035                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2036
2037                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2038                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2039                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2040                         routerInt, label, writeConfigTxn);
2041                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2042                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2043                         macAddress);
2044                 return;
2045             }
2046         }
2047         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2048                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
2049                 primaryRd, vpnName);
2050     }
2051
2052     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2053             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2054         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2055         String rd = vpnUtil.getVpnRd(vpnName);
2056         if (adjs != null) {
2057             List<Adjacency> adjsList = adjs.getAdjacency();
2058             for (Adjacency adj : adjsList) {
2059                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2060                     String primaryInterfaceIp = adj.getIpAddress();
2061                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2062                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2063                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2064                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2065                     return;
2066                 }
2067             }
2068         } else {
2069             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2070                     vpnInterface.getName(), rd);
2071         }
2072     }
2073
2074     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2075         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
2076             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
2077                   vpnName, intefaceData.vpnInterface.getName());
2078             return;
2079         }
2080         final VpnInterfaceKey key = intefaceData.identifier
2081                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
2082         final String interfaceName = key.getName();
2083         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2084                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2085         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2086                   intefaceData.identifier, vpnName);
2087         return;
2088     }
2089
2090     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2091                                               VpnInterface vpnInterface, String vpnName) {
2092         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2093                .get(vpnName);
2094         if (vpnInterfaces == null) {
2095             vpnInterfaces = new ConcurrentLinkedQueue<>();
2096         }
2097         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2098         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2099         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2100                 vpnInterface.getName(), vpnName);
2101     }
2102
2103     public boolean isVpnInstanceReady(String vpnInstanceName) {
2104         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2105         if (vpnRd == null) {
2106             return false;
2107         }
2108         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2109
2110         return vpnInstanceOpDataEntry != null;
2111     }
2112
2113     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2114         synchronized (vpnInstanceName.intern()) {
2115             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2116                     unprocessedVpnInterfaces.get(vpnInstanceName);
2117             if (vpnInterfaces != null) {
2118                 while (!vpnInterfaces.isEmpty()) {
2119                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2120                     if (hasVpnInstanceCreatedSuccessfully) {
2121                         processSavedInterface(savedInterface, vpnInstanceName);
2122                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2123                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2124                     } else {
2125                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2126                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2127                     }
2128                 }
2129             } else {
2130                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2131             }
2132         }
2133     }
2134
2135     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2136             VpnInterface vpnInterface) {
2137         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2138             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2139                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2140             if (vpnInterfaces != null) {
2141                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2142                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2143                             + "unprocessed list", vpnInterface.getName(),
2144                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2145                 }
2146             } else {
2147                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2148                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2149             }
2150         }
2151     }
2152
2153     public void vpnInstanceIsReady(String vpnInstanceName) {
2154         processSavedInterfaces(vpnInstanceName, true);
2155     }
2156
2157     public void vpnInstanceFailed(String vpnInstanceName) {
2158         processSavedInterfaces(vpnInstanceName, false);
2159     }
2160
2161     private static class UnprocessedVpnInterfaceData {
2162         InstanceIdentifier<VpnInterface> identifier;
2163         VpnInterface vpnInterface;
2164
2165         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2166             this.identifier = identifier;
2167             this.vpnInterface = vpnInterface;
2168         }
2169
2170         @Override
2171         public int hashCode() {
2172             final int prime = 31;
2173             int result = 1;
2174             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2175             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2176             return result;
2177         }
2178
2179         @Override
2180         public boolean equals(Object obj) {
2181             if (this == obj) {
2182                 return true;
2183             }
2184             if (obj == null) {
2185                 return false;
2186             }
2187             if (getClass() != obj.getClass()) {
2188                 return false;
2189             }
2190             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2191             if (identifier == null) {
2192                 if (other.identifier != null) {
2193                     return false;
2194                 }
2195             } else if (!identifier.equals(other.identifier)) {
2196                 return false;
2197             }
2198             if (vpnInterface == null) {
2199                 if (other.vpnInterface != null) {
2200                     return false;
2201                 }
2202             } else if (!vpnInterface.equals(other.vpnInterface)) {
2203                 return false;
2204             }
2205             return true;
2206         }
2207     }
2208
2209     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2210         String primaryRd = vpnUtil.getVpnRd(vpnName);
2211         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2212         if (vpnInstanceOpData == null) {
2213             return;
2214         }
2215         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2216         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2217             return;
2218         }
2219         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2220         vpnToDpnLists.forEach(vpnToDpnList -> {
2221             if (vpnToDpnList.getVpnInterfaces() == null) {
2222                 return;
2223             }
2224             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2225                 try {
2226                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2227                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2228                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2229                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2230                     if (!vpnInterfaceOptional.isPresent()) {
2231                         return;
2232                     }
2233                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2234                             vpnInterface.getInterfaceName());
2235                     if (configVpnAdjacencies == null) {
2236                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2237                         return;
2238                     }
2239                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2240                             .augmentation(AdjacenciesOp.class).getAdjacency();
2241                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2242                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2243                     // in operational DS. These unprocessed adjacencies will be handled below.
2244                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2245                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2246                     configVpnAdjacencies.stream()
2247                         .filter(adjacency -> operationVpnAdjacencies.stream()
2248                                 .noneMatch(operationalAdjacency ->
2249                                         operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2250                         .forEach(adjacency -> {
2251                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2252                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2253                                 () -> {
2254                                     // TODO Deal with sequencing — the config tx must only submitted
2255                                     // if the oper tx goes in
2256                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2257                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2258                                         futures.add(
2259                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx ->
2260                                                 futures.add(
2261                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2262                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2263                                                             primaryRd, adjacency, vpnInterfaceOptional.get()
2264                                                                 .getDpnId(), operTx, confTx, confTx)))));
2265                                         return futures;
2266                                     } else {
2267                                         return Collections.emptyList();
2268                                     }
2269                                 });
2270                         });
2271                 } catch (ReadFailedException e) {
2272                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2273                             vpnName, primaryRd);
2274                 }
2275             });
2276         });
2277     }
2278
2279     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2280         private final String interfaceName;
2281         private final boolean add;
2282         private final String txnDestination;
2283
2284         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2285             this.interfaceName = interfaceName;
2286             this.add = add;
2287             this.txnDestination = transactionDest;
2288         }
2289
2290         @Override
2291         public void onSuccess(Void voidObj) {
2292             if (add) {
2293                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2294                         interfaceName, txnDestination);
2295             } else {
2296                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2297             }
2298         }
2299
2300         @Override
2301         public void onFailure(Throwable throwable) {
2302             if (add) {
2303                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2304                         interfaceName, txnDestination, throwable);
2305             } else {
2306                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2307                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2308             }
2309         }
2310     }
2311 }