vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  42 import org.opendaylight.genius.mdsalutil.NWUtil;
  43 import org.opendaylight.genius.mdsalutil.NwConstants;
  44 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.infrautils.caches.CacheProvider;
  47 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  51 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  52 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  53 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  54 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  55 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  56 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  57 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  58 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  59 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 109 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 110 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 111 import org.slf4j.Logger;
 112 import org.slf4j.LoggerFactory;
 113
 114 @Singleton
 115 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 116
 117     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 118     private static final short DJC_MAX_RETRIES = 3;
 119
 120     private final DataBroker dataBroker;
 121     private final ManagedNewTransactionRunner txRunner;
 122     private final IBgpManager bgpManager;
 123     private final IFibManager fibManager;
 124     private final IMdsalApiManager mdsalManager;
 125     private final IdManagerService idManager;
 126     private final OdlInterfaceRpcService ifaceMgrRpcService;
 127     private final VpnFootprintService vpnFootprintService;
 128     private final IInterfaceManager interfaceManager;
 129     private final IVpnManager vpnManager;
 130     private final ArpResponderHandler arpResponderHandler;
 131     private final JobCoordinator jobCoordinator;
 132
 133     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 134
 135     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 136             new ConcurrentHashMap<>();
 137
 138     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 139
 140     @Inject
 141     public VpnInterfaceManager(final DataBroker dataBroker,
 142                                final IBgpManager bgpManager,
 143                                final IdManagerService idManager,
 144                                final IMdsalApiManager mdsalManager,
 145                                final IFibManager fibManager,
 146                                final OdlInterfaceRpcService ifaceMgrRpcService,
 147                                final VpnFootprintService vpnFootprintService,
 148                                final IInterfaceManager interfaceManager,
 149                                final IVpnManager vpnManager,
 150                                final ArpResponderHandler arpResponderHandler,
 151                                final JobCoordinator jobCoordinator,
 152                                final CacheProvider cacheProvider) {
 153         super(VpnInterface.class, VpnInterfaceManager.class);
 154
 155         this.dataBroker = dataBroker;
 156         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 157         this.bgpManager = bgpManager;
 158         this.idManager = idManager;
 159         this.mdsalManager = mdsalManager;
 160         this.fibManager = fibManager;
 161         this.ifaceMgrRpcService = ifaceMgrRpcService;
 162         this.vpnFootprintService = vpnFootprintService;
 163         this.interfaceManager = interfaceManager;
 164         this.vpnManager = vpnManager;
 165         this.arpResponderHandler = arpResponderHandler;
 166         this.jobCoordinator = jobCoordinator;
 167
 168         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 169                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 170                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 171     }
 172
 173     public Runnable isNotifyTaskQueued(String intfName) {
 174         return vpnIntfMap.remove(intfName);
 175     }
 176
 177     @PostConstruct
 178     public void start() {
 179         LOG.info("{} start", getClass().getSimpleName());
 180         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 181     }
 182
 183     @Override
 184     @PreDestroy
 185     public void close() {
 186         super.close();
 187         vpnInstanceOpDataEntryCache.close();
 188     }
 189
 190     @Override
 191     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 192         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 193     }
 194
 195     @Override
 196     protected VpnInterfaceManager getDataTreeChangeListener() {
 197         return VpnInterfaceManager.this;
 198     }
 199
 200     @Override
 201     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 202         LOG.info("add: intfName {} onto vpnName {}",
 203                  vpnInterface.getName(),
 204                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 205         addVpnInterface(identifier, vpnInterface, null, null);
 206     }
 207
 208     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 209                           final VpnInterface vpnInterface, String vpnName) {
 210         synchronized (vpnName.intern()) {
 211             if (isVpnInstanceReady(vpnName)) {
 212                 return true;
 213             }
 214             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 215             return false;
 216         }
 217     }
 218
 219     // TODO Clean up the exception handling
 220     @SuppressWarnings("checkstyle:IllegalCatch")
 221     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 222                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 223         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 224             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 225             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 226         }
 227     }
 228
 229     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 230                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 231         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 232         final String interfaceName = key.getName();
 233
 234         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 235             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 236                   vpnName, vpnInterface.getName());
 237             return;
 238         }
 239         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 240              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 241         List<Adjacency> copyOldAdjs = null;
 242         if (oldAdjs != null) {
 243             copyOldAdjs = new ArrayList<>();
 244             copyOldAdjs.addAll(oldAdjs);
 245         }
 246         List<Adjacency> copyNewAdjs = null;
 247         if (newAdjs != null) {
 248             copyNewAdjs = new ArrayList<>();
 249             copyNewAdjs.addAll(newAdjs);
 250         }
 251         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 252     }
 253
 254     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 255                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 256                     final List<Adjacency> newAdjs,
 257                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 258         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 259         final String interfaceName = key.getName();
 260         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 261         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 262             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 263             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 264             if (interfaceState != null) {
 265                 try {
 266                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 267                     final int ifIndex = interfaceState.getIfIndex();
 268                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 269                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 270                         // (the inventory tx goes in last)
 271                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 272                         ListenableFuture<Void> confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 273                             confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 274                                 operTx -> futures.add(
 275                                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> {
 276                                             LOG.info(
 277                                                     "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 278                                                     + " on dpn {}",
 279                                                     vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 280                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 281                                                     confTx, operTx, invTx, interfaceState, vpnName);
 282                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 283                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 284                                                         + " Update for swapping VPN {} case.", interfaceName, vpnName);
 285                                                 if (newAdjs != null) {
 286                                                     for (Adjacency adj : newAdjs) {
 287                                                         if (oldAdjs.contains(adj)) {
 288                                                             oldAdjs.remove(adj);
 289                                                         } else {
 290                                                             if (!isBgpVpnInternetVpn
 291                                                                     || VpnUtil.isAdjacencyEligibleToVpnInternet(
 292                                                                     dataBroker, adj)) {
 293                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 294                                                                         primaryRd, adj, dpnId, operTx, confTx);
 295                                                             }
 296                                                         }
 297                                                     }
 298                                                 }
 299                                                 for (Adjacency adj : oldAdjs) {
 300                                                     if (!isBgpVpnInternetVpn
 301                                                             || VpnUtil.isAdjacencyEligibleToVpnInternet(
 302                                                             dataBroker, adj)) {
 303                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 304                                                                 operTx, confTx);
 305                                                     }
 306                                                 }
 307                                             }
 308                                         })))));
 309                         futures.add(confFuture);
 310                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 311                                 MoreExecutors.directExecutor());
 312                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 313                                 + " processed successfully", interfaceName, vpnName, dpnId);
 314                         return futures;
 315                     });
 316                 } catch (NumberFormatException | IllegalStateException e) {
 317                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 318                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 319                                     vpnName, e);
 320                     return;
 321                 }
 322             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 323                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 324                     () -> {
 325                         ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
 326                             createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 327                                     confTx, vpnName);
 328                             LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 329                                     vpnName, vpnInterface.getDpnId());
 330                         });
 331                         ListenableFutures.addErrorLogging(future, LOG,
 332                                 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 333                         return Collections.singletonList(future);
 334                     });
 335             } else {
 336                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 337                         + " is not available", interfaceName, vpnName);
 338             }
 339         } else {
 340             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 341                       + " as vpn is pending delete", interfaceName, vpnName,
 342                     vpnInterface.getDpnId());
 343         }
 344     }
 345
 346     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 347     // see comments below.
 348     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 349     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 350             final int lportTag, boolean isInterfaceUp,
 351             WriteTransaction writeConfigTxn,
 352             WriteTransaction writeOperTxn,
 353             WriteTransaction writeInvTxn,
 354             Interface interfaceState,
 355             final String vpnName) {
 356         final String interfaceName = vpnInterface.getName();
 357         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 358                                                          interfaceName, vpnName);
 359         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 360         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 361         if (!isInterfaceUp) {
 362             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 363                      interfaceName, dpId, vpnName);
 364             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 365             if (vpnId == VpnConstants.INVALID_ID) {
 366                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 367                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 368                         dpId);
 369                 return;
 370             }
 371
 372             boolean waitForVpnInterfaceOpRemoval = false;
 373             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 374                 String opVpnName = opVpnInterface.getVpnInstanceName();
 375                 String primaryInterfaceIp = null;
 376                 if (opVpnName.equals(vpnName)) {
 377                     // Please check if the primary VRF Entry does not exist for VPNInterface
 378                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 379                     // back to back
 380                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 381                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 382                     if (adjs == null) {
 383                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 384                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 385                                 vpnName);
 386                         return;
 387                     }
 388                     for (Adjacency adj : adjs) {
 389                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 390                             primaryInterfaceIp = adj.getIpAddress();
 391                             break;
 392                         }
 393                     }
 394                     if (primaryInterfaceIp == null) {
 395                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 396                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 397                                 dpId, vpnName);
 398                         return;
 399                     }
 400                     // Get the rd of the vpn instance
 401                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 402                     if (vrf != null) {
 403                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 404                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 405                         return;
 406                     }
 407                     waitForVpnInterfaceOpRemoval = true;
 408                 } else {
 409                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 410                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 411                 }
 412             }
 413             if (!waitForVpnInterfaceOpRemoval) {
 414                 // Add the VPNInterface and quit
 415                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 416                         null/*ipAddressSourceValuePair*/,
 417                         true /* add */);
 418                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 419                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 420                 if (!isBgpVpnInternetVpn) {
 421                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 422                             jobCoordinator);
 423                 }
 424                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 425                         dpId, vpnName);
 426                 if (interfaceManager.isExternalInterface(interfaceName)) {
 427                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 428                         NwConstants.ADD_FLOW);
 429                 }
 430                 return;
 431             }
 432
 433             // FIB didn't get a chance yet to clean up this VPNInterface
 434             // Let us give it a chance here !
 435             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 436                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 437             try {
 438                 Runnable notifyTask = new VpnNotifyTask();
 439                 synchronized (notifyTask) {
 440                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 441                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 442                     // published here while holding the lock on it so this path will hit the wait before notify can be
 443                     // invoked.
 444                     vpnIntfMap.put(interfaceName, notifyTask);
 445                     try {
 446                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 447                     } catch (InterruptedException e) {
 448                         // Ignored
 449                     }
 450                 }
 451             } finally {
 452                 vpnIntfMap.remove(interfaceName);
 453             }
 454
 455             if (opVpnInterface != null) {
 456                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 457                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 458                         dpId, vpnName);
 459                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
 460                 return;
 461             }
 462             // VPNInterface got removed, proceed with Add
 463             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 464                     interfaceName, dpId, vpnName);
 465             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 466                     null/*ipAddressSourceValuePair*/,
 467                     true /* add */);
 468             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 469                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 470             if (!isBgpVpnInternetVpn) {
 471                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 472             }
 473             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 474                     + " FIB to clean up", interfaceName, dpId, vpnName);
 475             if (interfaceManager.isExternalInterface(interfaceName)) {
 476                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 477                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 478             }
 479         } else {
 480             // Interface is retained in the DPN, but its Link Up.
 481             // Advertise prefixes again for this interface to BGP
 482             InstanceIdentifier<VpnInterface> identifier =
 483                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 484             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 485                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 486             advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 487             // Perform similar operation as interface add event for extraroutes.
 488             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 489             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 490             if (!optAdjacencies.isPresent()) {
 491                 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 492                 return;
 493             }
 494             List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 495             for (Adjacency adjacency : adjacencies) {
 496                 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 497                     continue;
 498                 }
 499                 // if BGPVPN Internet, filter only IPv6 Adjacencies
 500                 if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 501                                 dataBroker, adjacency)) {
 502                     continue;
 503                 }
 504                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 505                           dpId, writeOperTxn, writeConfigTxn);
 506             }
 507         }
 508     }
 509
 510     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 511             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 512         Uuid extNetworkId;
 513         try {
 514             // vpn instance of ext-net interface is the network-id
 515             extNetworkId = new Uuid(vpnName);
 516         } catch (IllegalArgumentException e) {
 517             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 518                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 519             return;
 520         }
 521
 522         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 523         if (routerIds == null || routerIds.isEmpty()) {
 524             LOG.info("processExternalVpnInterface: No router is associated with {}."
 525                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 526                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 527             return;
 528         }
 529
 530         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 531                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 532         for (Uuid routerId : routerIds) {
 533             String routerName = routerId.getValue();
 534             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 535             if (Objects.equals(primarySwitch, dpId)) {
 536                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 537                 if (router != null) {
 538                     if (addOrRemove == NwConstants.ADD_FLOW) {
 539                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 540                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 541                                 dpId, interfaceName, lportTag, writeInvTxn);
 542                     } else {
 543                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 544                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 545                                 dpId, interfaceName, lportTag);
 546                     }
 547                 } else {
 548                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 549                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 550                             interfaceName, dpId, vpnName);
 551                 }
 552             }
 553         }
 554     }
 555
 556     // TODO Clean up the exception handling
 557     @SuppressWarnings("checkstyle:IllegalCatch")
 558     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 559                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 560                                                   String vpnName, String interfaceName) {
 561         if (rd == null) {
 562             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 563                   interfaceName, dpnId, vpnName);
 564             return;
 565         } else {
 566             if (rd.equals(vpnName)) {
 567                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 568                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 569                 return;
 570             }
 571         }
 572         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 573                 interfaceName, dpnId, vpnName, rd);
 574
 575         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 576         if (nextHopIp == null) {
 577             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 578                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 579                     rd, vpnName);
 580             return;
 581         }
 582
 583         //Read NextHops
 584         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 585         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 586         if (adjacencies.isPresent()) {
 587             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 588
 589             if (!nextHops.isEmpty()) {
 590                 LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 591                         + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 592                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 593                 long l3vni = vpnInstanceOpData.getL3vni();
 594                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 595                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 596                 for (Adjacency nextHop : nextHops) {
 597                     if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 598                         continue;
 599                     }
 600                     String gatewayMac = null;
 601                     long label = 0;
 602                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 603                         final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 604                             vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 605                         gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName).get();
 606                     } else {
 607                         label = nextHop.getLabel();
 608                     }
 609                     try {
 610                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 611                                 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 612                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 613                                 encapType, (int)label, l3vni, 0 /*l2vni*/,
 614                                 gatewayMac);
 615                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 616                                 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 617                                 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 618                     } catch (Exception e) {
 619                         LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}"
 620                                 + " for interface {} on dpn {}", nextHop.getIpAddress(),
 621                                   vpnName, rd, interfaceName, dpnId, e);
 622                     }
 623                 }
 624             }
 625         }
 626     }
 627
 628     // TODO Clean up the exception handling
 629     @SuppressWarnings("checkstyle:IllegalCatch")
 630     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 631                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 632         //Read NextHops
 633         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 634         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 635
 636         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 637         if (rd == null) {
 638             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 639                 interfaceName, vpnName);
 640             return;
 641         } else {
 642             if (rd.equals(vpnName)) {
 643                 LOG.info(
 644                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 645                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 646                 return;
 647             }
 648         }
 649         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 650                vpnName, rd);
 651         if (adjacencies.isPresent()) {
 652             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 653
 654             if (!nextHops.isEmpty()) {
 655                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 656                         nextHops, interfaceName, vpnName, rd);
 657                 for (Adjacency nextHop : nextHops) {
 658                     try {
 659                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 660                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 661                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 662                                     interfaceName, vpnName);
 663                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 664                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 665                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 666                                     interfaceName, vpnName);
 667                         } else {
 668                             // Perform similar operation as interface delete event for extraroutes.
 669                             String allocatedRd = nextHop.getVrfId();
 670                             for (String nh : nextHop.getNextHopIpList()) {
 671                                 deleteExtraRouteFromCurrentAndImportingVpns(
 672                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 673                             }
 674                         }
 675                     } catch (Exception e) {
 676                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 677                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 678                     }
 679                 }
 680             }
 681         }
 682     }
 683
 684     @SuppressWarnings("checkstyle:IllegalCatch")
 685     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 686                                                   String primaryRd, String interfaceName, final long vpnId,
 687                                                   WriteTransaction writeConfigTxn,
 688                                                   WriteTransaction writeOperTxn,
 689                                                   final WriteTransaction writeInvTxn,
 690                                                   Interface interfaceState) {
 691         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 692         // Read NextHops
 693         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 694         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 695         if (!adjacencies.isPresent()) {
 696             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 697                     null/*gwMac*/, writeOperTxn);
 698             return;
 699         }
 700
 701         // Get the rd of the vpn instance
 702         String nextHopIp = null;
 703         try {
 704             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 705         } catch (Exception e) {
 706             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 707                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 708         }
 709         List<String> nhList = new ArrayList<>();
 710         if (nextHopIp != null) {
 711             nhList.add(nextHopIp);
 712             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 713                     interfaceName, dpnId, vpnName, nhList);
 714         }
 715         Optional<String> gwMac = Optional.absent();
 716         String vpnInterfaceSubnetGwMacAddress = null;
 717         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 718         Long l3vni = vpnInstanceOpData.getL3vni();
 719         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 720         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 721         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 722         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 723         List<Adjacency> value = new ArrayList<>();
 724         for (Adjacency nextHop : nextHops) {
 725             String rd = primaryRd;
 726             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 727             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 728                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 729                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 730                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 731                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 732                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 733                 continue;
 734             }
 735             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 736                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 737                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 738                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 739                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 740                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 741                 writeOperTxn.merge(
 742                     LogicalDatastoreType.OPERATIONAL,
 743                     VpnUtil.getPrefixToInterfaceIdentifier(
 744                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 745                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 746                             prefixCue), true);
 747                 final Uuid subnetId = nextHop.getSubnetId();
 748
 749                 String gatewayIp = nextHop.getSubnetGatewayIp();
 750                 if (gatewayIp == null) {
 751                     Optional<String> gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 752                     if (gatewayIpOptional.isPresent()) {
 753                         gatewayIp = gatewayIpOptional.get();
 754                     }
 755                 }
 756
 757                 if (gatewayIp != null) {
 758                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 759                     if (gwMac.isPresent()) {
 760                         // A valid mac-address is available for this subnet-gateway-ip
 761                         // Use this for programming ARP_RESPONDER table here.  And save this
 762                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 763                         // to populate L3_GW_MAC_TABLE there.
 764                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 765                                 gatewayIp, gwMac.get());
 766                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 767                     } else {
 768                         // A valid mac-address is not available for this subnet-gateway-ip
 769                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 770                         // Save this connected-mac-address as gateway-mac-address for the
 771                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 772                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 773                         if (gwMac.isPresent()) {
 774                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 775                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 776                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 777                                     gatewayIp, gwMac.get());
 778                         } else {
 779                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 780                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 781                                 + "gwIp {}",
 782                                 subnetId, interfaceName, vpnName, gatewayIp);
 783                         }
 784                     }
 785                 } else {
 786                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 787                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 788                         subnetId, interfaceName, vpnName);
 789                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 790                 }
 791                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 792                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 793             } else {
 794                 //Extra route adjacency
 795                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 796                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 797                 synchronized (vpnPrefixKey.intern()) {
 798                     java.util.Optional<String> rdToAllocate = VpnUtil
 799                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 800                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 801                     if (rdToAllocate.isPresent()) {
 802                         rd = rdToAllocate.get();
 803                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 804                             rd, prefix);
 805                     } else {
 806                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 807                         continue;
 808                     }
 809                 }
 810                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 811                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 812                         interfaceName, dpnId);
 813             }
 814             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 815             // can be different from the gateway-mac-address within the VRFEntry as the
 816             // gateway-mac-address is a superset.
 817             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 818                     : RouteOrigin.STATIC;
 819             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 820                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 821                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 822             Adjacency operationalAdjacency = null;
 823             try {
 824                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 825             } catch (NullPointerException e) {
 826                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 827                     input, e.getMessage());
 828                 return;
 829             }
 830             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 831                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 832                         vpnName, l3vni, origin,
 833                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 834             }
 835             value.add(operationalAdjacency);
 836         }
 837
 838         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 839         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 840                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 841
 842         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 843                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 844                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 845
 846         for (Adjacency nextHop : aug.getAdjacency()) {
 847             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 848             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 849                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 850                         : RouteOrigin.STATIC;
 851                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 852                 registeredPopulator.populateFib(input, writeConfigTxn);
 853             }
 854         }
 855     }
 856
 857     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 858                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 859         VpnInterfaceOpDataEntry opInterface =
 860               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 861         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 862             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 863         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 864                 WriteTransaction.CREATE_MISSING_PARENTS);
 865         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 866                 interfaceName, dpnId, vpnName);
 867     }
 868
 869     // TODO Clean up the exception handling
 870     @SuppressWarnings("checkstyle:IllegalCatch")
 871     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 872                                            StateTunnelList stateTunnelList,
 873                                            WriteTransaction writeConfigTxn,
 874                                            WriteTransaction writeOperTxn) {
 875
 876         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 877         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 878         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 879         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 880         if (adjList.isEmpty()) {
 881             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 882                     vpnInterface, srcDpnId);
 883             return;
 884         }
 885         String prefix = null;
 886         long label = 0;
 887         List<Adjacency> value = new ArrayList<>();
 888         boolean isNextHopAddReqd = false;
 889         String vpnName = vpnInterface.getVpnInstanceName();
 890         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 891         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 892         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 893                 vpnInterface.getName(), vpnInterface.getDpnId(),
 894                 vpnInterface.getVpnInstanceName(), adjList);
 895         for (Adjacency adj : adjList) {
 896             String rd = adj.getVrfId();
 897             rd = rd != null ? rd : vpnName;
 898             prefix = adj.getIpAddress();
 899             label = adj.getLabel();
 900             List<String> nhList = Collections.singletonList(srcTepIp);
 901             List<String> nextHopList = adj.getNextHopIpList();
 902             // If TEP is added , update the nexthop of primary adjacency.
 903             // Secondary adj nexthop is already pointing to primary adj IP address.
 904             if (nextHopList != null && !nextHopList.isEmpty()) {
 905                 /* everything right already */
 906             } else {
 907                 isNextHopAddReqd = true;
 908             }
 909
 910             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 911                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 912             } else {
 913                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 914                 if (!vrfEntryOptional.isPresent()) {
 915                     continue;
 916                 }
 917                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 918                 if (!nhList.contains(srcTepIp)) {
 919                     nhList.add(srcTepIp);
 920                     isNextHopAddReqd = true;
 921                 }
 922                 value.add(adj);
 923             }
 924
 925             if (isNextHopAddReqd) {
 926                 updateLabelMapper(label, nhList);
 927                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 928                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 929                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 930                 // Update the VRF entry with nextHop
 931                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 932                         label, true, writeConfigTxn);
 933
 934                 //Get the list of VPN's importing this route(prefix) .
 935                 // Then update the VRF entry with nhList
 936                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 937                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 938                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 939                     String vpnRd = vpn.getVrfId();
 940                     if (vpnRd != null) {
 941                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 942                                 srcTepIp, label, true, writeConfigTxn);
 943                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 944                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 945                             vpnInterface.getName(), srcDpnId, vpnName,
 946                             vpn.getVpnInstanceName(), vpnRd);
 947                     }
 948                 }
 949                 // Advertise the prefix to BGP only for external vpn
 950                 // since there is a nexthop change.
 951                 try {
 952                     if (!rd.equalsIgnoreCase(vpnName)) {
 953                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 954                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 955                                 null /*gatewayMacAddress*/);
 956                     }
 957                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 958                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 959                             srcDpnId, vpnName);
 960                 } catch (Exception ex) {
 961                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 962                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 963                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 964                 }
 965             }
 966         }
 967         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 968         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 969                 .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 970                 .addAugmentation(AdjacenciesOp.class, aug).build();
 971         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 972                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 973         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 974                 WriteTransaction.CREATE_MISSING_PARENTS);
 975         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 976                 vpnInterface.getName(), srcDpnId, vpnName);
 977
 978     }
 979
 980     // TODO Clean up the exception handling
 981     @SuppressWarnings("checkstyle:IllegalCatch")
 982     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
 983                                               StateTunnelList stateTunnelList,
 984                                               WriteTransaction writeConfigTxn,
 985                                               WriteTransaction writeOperTxn) {
 986
 987         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 988         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 989         String prefix = null;
 990         long label = 0;
 991         boolean isNextHopRemoveReqd = false;
 992         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 993         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 994         String vpnName = vpnInterface.getVpnInstanceName();
 995         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 996         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 997         if (adjList != null) {
 998             List<Adjacency> value = new ArrayList<>();
 999             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1000                     vpnInterface.getName(), vpnInterface.getDpnId(),
1001                     vpnInterface.getVpnInstanceName(), adjList);
1002             for (Adjacency adj : adjList) {
1003                 List<String> nhList = new ArrayList<>();
1004                 String rd = adj.getVrfId();
1005                 rd = rd != null ? rd : vpnName;
1006                 prefix = adj.getIpAddress();
1007                 List<String> nextHopList = adj.getNextHopIpList();
1008                 label = adj.getLabel();
1009                 if (nextHopList != null && !nextHopList.isEmpty()) {
1010                     isNextHopRemoveReqd = true;
1011                 }
1012                 // If TEP is deleted , remove the nexthop from primary adjacency.
1013                 // Secondary adj nexthop will continue to point to primary adj IP address.
1014                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1015                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1016                 } else {
1017                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1018                     if (!vrfEntryOptional.isPresent()) {
1019                         continue;
1020                     }
1021                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1022                     if (nhList.contains(srcTepIp)) {
1023                         nhList.remove(srcTepIp);
1024                         isNextHopRemoveReqd = true;
1025                     }
1026                     value.add(adj);
1027                 }
1028
1029                 if (isNextHopRemoveReqd) {
1030                     updateLabelMapper(label, nhList);
1031                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1032                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1033                             prefix, nhList, vpnName,
1034                             vpnId, rd, vpnInterface.getName());
1035                     // Update the VRF entry with removed nextHop
1036                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1037                             label, false, writeConfigTxn);
1038
1039                     //Get the list of VPN's importing this route(prefix) .
1040                     // Then update the VRF entry with nhList
1041                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1042                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1043                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1044                         String vpnRd = vpn.getVrfId();
1045                         if (vpnRd != null) {
1046                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1047                                     srcTepIp, label, false, writeConfigTxn);
1048                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1049                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1050                                     nhList, label, vpnInterface.getName(), srcDpnId,
1051                                     vpnName,
1052                                     vpn.getVpnInstanceName(), vpnRd);
1053                         }
1054                     }
1055
1056                     // Withdraw prefix from BGP only for external vpn.
1057                     try {
1058                         if (!rd.equalsIgnoreCase(vpnName)) {
1059                             bgpManager.withdrawPrefix(rd, prefix);
1060                         }
1061                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1062                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1063                                 vpnInterface.getName(), srcDpnId,
1064                                 vpnName);
1065                     } catch (Exception ex) {
1066                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1067                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1068                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1069                     }
1070                 }
1071             }
1072             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1073             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1074                     .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1075                     .addAugmentation(AdjacenciesOp.class, aug).build();
1076             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1077                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1078             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1079                     WriteTransaction.CREATE_MISSING_PARENTS);
1080             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1081                          vpnInterface.getName(), srcDpnId, vpnName);
1082         }
1083     }
1084
1085     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1086         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1087
1088         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1089         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1090         if (vpnInstanceOpDataEntry == null) {
1091             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1092                     + " to check for vpns exporting the routes", vpnName);
1093             return vpnsToExportRoute;
1094         }
1095
1096         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1097             if (input.getVpnInstanceName() == null) {
1098                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1099                         input.getVrfId());
1100                 return false;
1101             }
1102             return !input.getVpnInstanceName().equals(vpnName);
1103         };
1104
1105         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1106             Iterable<String> commonRTs =
1107                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1108                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1109             return Iterators.size(commonRTs.iterator()) > 0;
1110         };
1111
1112         vpnsToExportRoute =
1113                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1114                         Collectors.toList());
1115         return vpnsToExportRoute;
1116     }
1117
1118     // TODO Clean up the exception handling
1119     @SuppressWarnings("checkstyle:IllegalCatch")
1120     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1121         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1122         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1123             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1124             if (vrfEntries != null) {
1125                 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1126                     for (VrfEntry vrfEntry : vrfEntries) {
1127                         try {
1128                             if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1129                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1130                                 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1131                                                 + " is not a controller managed non intervpn link route. Ignoring.",
1132                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1133                                 continue;
1134                             }
1135                             String prefix = vrfEntry.getDestPrefix();
1136                             String gwMac = vrfEntry.getGatewayMacAddress();
1137                             vrfEntry.getRoutePaths().forEach(routePath -> {
1138                                 String nh = routePath.getNexthopAddress();
1139                                 int label = routePath.getLabel().intValue();
1140                                 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1141                                         vrfEntry.getOrigin()))) {
1142                                     LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1143                                                     + " nexthop {} label {} to vpn {} vpnRd {}",
1144                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1145                                     fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1146                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1147                                             0 /*l3vni*/, gwMac,  vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1148                                             confTx);
1149                                 } else {
1150                                     LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1151                                                     + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1152                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1153                                     SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1154                                     importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1155                                             confTx);
1156                                 }
1157                             });
1158                         } catch (RuntimeException e) {
1159                             LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1160                                             + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1161                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1162                         }
1163                     }
1164                 }), LOG, "Error handing VPN exporting routes");
1165             } else {
1166                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1167                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1168             }
1169         }
1170     }
1171
1172     @Override
1173     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1174         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1175         final String interfaceName = key.getName();
1176         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1177             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1178             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1179         }
1180     }
1181
1182     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1183                                 final VpnInterface vpnInterface, final String vpnName,
1184                                 final String interfaceName) {
1185         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1186             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1187                 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1188                     deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1189                     LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1190                 });
1191                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1192                         vpnInterface.getName(), vpnName);
1193                 return Collections.singletonList(future);
1194             }, DJC_MAX_RETRIES);
1195         } else {
1196             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1197             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1198         }
1199     }
1200
1201     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1202     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1203                                 final VpnInterface vpnInterface, final String vpnName,
1204                                 final String interfaceName, final Interface interfaceState) {
1205         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1206                 vpnName, vpnInterface.getDpnId());
1207         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1208         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1209             () -> {
1210                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1211                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1212                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1213                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1214                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1215                                     interfaceName, vpnName);
1216                             BigInteger dpId = BigInteger.ZERO;
1217                             int ifIndex = 0;
1218                             String gwMacAddress = null;
1219                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1220                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1221                             final Optional<VpnInterfaceOpDataEntry> optVpnInterface =
1222                                     VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1223                             if (interfaceState != null) {
1224                                 try {
1225                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1226                                 } catch (NumberFormatException | IllegalStateException e) {
1227                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1228                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1229                                                     + " from vpn interface op data store. ", interfaceName,
1230                                             vpnInterface.getDpnId(), vpnName, e);
1231                                     dpId = BigInteger.ZERO;
1232                                 }
1233                                 ifIndex = interfaceState.getIfIndex();
1234                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1235                             } else {
1236                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1237                                         + " from vpn interface op.", interfaceName);
1238                                 if (optVpnInterface.isPresent()) {
1239                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1240                                     dpId = vpnOpInterface.getDpnId();
1241                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1242                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1243                                 } else {
1244                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1245                                                     + " as interfaceState and vpn interface op is not"
1246                                             + " available", interfaceName, vpnName);
1247                                     return;
1248                                 }
1249                             }
1250                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1251                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1252                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1253                             LOG.info(
1254                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1255                                             + "successfully",
1256                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1257                         })))));
1258                 futures.add(configFuture);
1259                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1260                 return futures;
1261             }, DJC_MAX_RETRIES);
1262     }
1263
1264     protected void processVpnInterfaceDown(BigInteger dpId,
1265                                            String interfaceName,
1266                                            int lportTag,
1267                                            String gwMac,
1268                                            VpnInterfaceOpDataEntry vpnOpInterface,
1269                                            boolean isInterfaceStateDown,
1270                                            WriteTransaction writeConfigTxn,
1271                                            WriteTransaction writeOperTxn,
1272                                            WriteTransaction writeInvTxn) {
1273         if (vpnOpInterface == null) {
1274             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1275                     + " as it is not available in operational data store", interfaceName, dpId);
1276             return;
1277         }
1278         final String vpnName = vpnOpInterface.getVpnInstanceName();
1279         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1280                                                     interfaceName, vpnName);
1281         if (!isInterfaceStateDown) {
1282             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1283             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1284                     null);
1285             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1286             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1287                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1288             if (interfaceManager.isExternalInterface(interfaceName)) {
1289                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1290                         NwConstants.DEL_FLOW);
1291             }
1292             if (!isBgpVpnInternetVpn) {
1293                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1294             }
1295             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1296                     + " successful", interfaceName, dpId, vpnName);
1297         } else {
1298             // Interface is retained in the DPN, but its Link Down.
1299             // Only withdraw the prefixes for this interface from BGP
1300             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1301         }
1302     }
1303
1304     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1305                                           final String vpnName, final long vpnId, String gwMac,
1306                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1307                                           final WriteTransaction writeInvTxn) {
1308         //Read NextHops
1309         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1310                 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1311         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1312         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1313
1314         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1315         LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1316                 interfaceName, dpnId, vpnName, primaryRd);
1317         if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1318             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1319             LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1320                     interfaceName, dpnId, vpnName, nextHops);
1321             for (Adjacency nextHop : nextHops) {
1322                 if (nextHop.isPhysNetworkFunc()) {
1323                     LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1324                             nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1325                     fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1326                             null/*writeCfgTxn*/);
1327                 } else {
1328                     String rd = nextHop.getVrfId();
1329                     List<String> nhList;
1330                     if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1331                         nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1332                     } else {
1333                         // This is a primary adjacency
1334                         nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1335                                 : Collections.emptyList();
1336                         removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1337                                 interfaceName, writeInvTxn);
1338                     }
1339                     if (!nhList.isEmpty()) {
1340                         if (rd.equals(vpnName)) {
1341                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1342                             //remove from FIB directly
1343                             nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1344                                     interfaceName, dpnId, writeConfigTxn));
1345                         } else {
1346                             removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, interfaceName,
1347                                     writeConfigTxn);
1348                         }
1349                     } else {
1350                         LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1351                                         + " interface {}", nextHop.getIpAddress(), rd,
1352                                 nextHop.getAdjacencyType().toString(), interfaceName);
1353                         bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1354                         fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1355                     }
1356                 }
1357                 String ip = nextHop.getIpAddress().split("/")[0];
1358                 LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1359                 if (vpnVipToPort != null) {
1360                     VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip, null);
1361                     LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1362                                     + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(),
1363                             ip, dpnId, vpnName);
1364                 }
1365             }
1366         } else {
1367             // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1368             LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1369                     + " Removing it.", interfaceName, vpnName, dpnId);
1370             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1371         }
1372     }
1373
1374     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1375                                                                    String interfaceName, BigInteger dpnId,
1376                                                                    WriteTransaction writeConfigTxn) {
1377         return (nh) -> {
1378             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1379                     writeConfigTxn);
1380             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1381                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1382                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1383         };
1384     }
1385
1386     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1387                                            BigInteger dpnId, String rd, String interfaceName,
1388                                            WriteTransaction writeConfigTxn) {
1389         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1390                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1391         nhList.forEach((nh) -> {
1392             //IRT: remove routes from other vpns importing it
1393             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1394                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1395             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1396                 String vpnRd = vpn.getVrfId();
1397                 if (vpnRd != null) {
1398                     fibManager.removeOrUpdateFibEntry(vpnRd,
1399                             nextHop.getIpAddress(), nh, writeConfigTxn);
1400                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1401                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1402                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1403                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1404                 }
1405             }
1406         });
1407     }
1408
1409     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1410                                                  int lportTag, String gwMac, String interfaceName,
1411                                                  WriteTransaction writeInvTxn) {
1412         final Uuid subnetId = nextHop.getSubnetId();
1413         if (nextHop.getSubnetGatewayMacAddress() == null) {
1414             // A valid mac-address was not available for this subnet-gateway-ip
1415             // So a connected-mac-address was used for this subnet and we need
1416             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1417             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1418                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1419         }
1420         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1421                 subnetId);
1422     }
1423
1424     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1425                                                   String interfaceName) {
1426         // This is either an extra-route (or) a learned IP via subnet-route
1427         List<String> nhList = null;
1428         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1429         if (nextHopIp == null || nextHopIp.isEmpty()) {
1430             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1431                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1432                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1433                     vpnName);
1434             nhList = Collections.emptyList();
1435         } else {
1436             nhList = Collections.singletonList(nextHopIp);
1437         }
1438         return nhList;
1439     }
1440
1441     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1442         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1443         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1444         // else use connected interface
1445         if (gwPort != null && gwPort.isSubnetIp()) {
1446             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1447                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1448             return Optional.of(gwPort.getMacAddress());
1449         }
1450         return Optional.absent();
1451     }
1452
1453     @Override
1454     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1455         final VpnInterface update) {
1456         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1457                 update.getDpnId(), original.getVpnInstanceNames(),
1458                 update.getVpnInstanceNames());
1459         final String vpnInterfaceName = update.getName();
1460         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1461         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1462         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1463             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1464         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1465         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1466             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1467
1468         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1469         //handles switching between <internal VPN - external VPN>
1470         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1471             LOG.info("update: handled VPNInterface {} on dpn {} update"
1472                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1473                      original.getName(), dpnId,
1474                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1475                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1476             return;
1477         }
1478         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1479             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1480             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1481             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1482             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1483             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1484                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1485                     // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1486                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1487                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1488                         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx -> {
1489                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1490                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1491                             LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1492                                     update.getName(), newVpnName);
1493                             //handle both addition and removal of adjacencies
1494                             //currently, new adjacency may be an extra route
1495                             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1496                             if (!oldAdjs.equals(newAdjs)) {
1497                                 for (Adjacency adj : copyNewAdjs) {
1498                                     if (copyOldAdjs.contains(adj)) {
1499                                         copyOldAdjs.remove(adj);
1500                                     } else {
1501                                         // add new adjacency - right now only extra route will hit this path
1502                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker,
1503                                                 adj)) {
1504                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1505                                                     dpnId, operTx, confTx);
1506                                         }
1507                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1508                                                 + " vpn interface {} on vpn {} dpnId {}",
1509                                                 adj.getIpAddress(), adj.getNextHopIpList(),
1510                                                 adj.getLabel(), adj.getSubnetId(), update.getName(),
1511                                                 newVpnName, dpnId);
1512                                     }
1513                                 }
1514                                 for (Adjacency adj : copyOldAdjs) {
1515                                     if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker,
1516                                             adj)) {
1517                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1518                                                 && !adj.isPhysNetworkFunc()) {
1519                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1520                                                     operTx, confTx);
1521                                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1522                                                     LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier);
1523                                             VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get();
1524                                             long vpnId = VpnUtil.getVpnId(dataBroker, newVpnName);
1525                                             VpnUtil.removePrefixToInterfaceAdj(dataBroker, adj, vpnId,
1526                                                     vpnInterfaceOpDataEntry, operTx);
1527                                             //remove FIB entry
1528                                             String vpnRd = VpnUtil.getVpnRd(dataBroker, newVpnName);
1529                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1530                                                     + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1531                                             //remove BGP entry
1532                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1533                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1534                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1535                                             }
1536                                         } else {
1537                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1538                                                     operTx, confTx);
1539                                         }
1540                                     }
1541                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1542                                                     + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1543                                                     .getNextHopIpList(),
1544                                             adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1545                                 }
1546                             }
1547                         }));
1548                     }));
1549                     return futures;
1550                 });
1551             } else {
1552                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1553                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1554             }
1555         }
1556     }
1557
1558     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1559                                                  VpnInterface original, VpnInterface update) {
1560         boolean isSwap = Boolean.FALSE;
1561         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1562         final String interfaceName = key.getName();
1563         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1564             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1565         List<String> oldVpnListCopy = new ArrayList<>();
1566         oldVpnListCopy.addAll(oldVpnList);
1567         List<String> newVpnList = update.getVpnInstanceNames().stream()
1568             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1569         oldVpnList.removeAll(newVpnList);
1570         newVpnList.removeAll(oldVpnListCopy);
1571         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1572             for (String oldVpnName: oldVpnList) {
1573                 isSwap = Boolean.TRUE;
1574                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1575                         + " running config-driven swap removal", interfaceName, oldVpnName);
1576                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1577                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1578                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1579             }
1580             //Wait for previous interface bindings to be removed
1581             try {
1582                 Thread.sleep(2000);
1583             } catch (InterruptedException e) {
1584                 //Ignore
1585             }
1586             for (String newVpnName: newVpnList) {
1587                 String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1588                 isSwap = Boolean.TRUE;
1589                 if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1590                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1591                             + "running config-driven swap addition", interfaceName, newVpnName);
1592                     final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1593                     final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1594                             ? origAdjs.getAdjacency() : new ArrayList<>();
1595                     final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1596                     final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1597                             ? updateAdjs.getAdjacency() : new ArrayList<>();
1598
1599                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1600                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1601                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1602                             interfaceName, oldVpnListCopy, newVpnName);
1603                 }
1604             }
1605         }
1606         return isSwap;
1607     }
1608
1609     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1610         Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1611         synchronized (label.toString().intern()) {
1612             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1613                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1614             Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1615             if (opResult.isPresent()) {
1616                 LabelRouteInfo labelRouteInfo =
1617                     new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1618                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1619             }
1620         }
1621         LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList);
1622     }
1623
1624     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1625         SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1626
1627         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1628         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1629                 .addAugmentation(SubnetRoute.class, route).build();
1630         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1631         InstanceIdentifierBuilder<VrfTables> idBuilder =
1632             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1633         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1634         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1635         if (writeConfigTxn != null) {
1636             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1637         } else {
1638             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1639         }
1640         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1641                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1642     }
1643
1644     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1645                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1646                                            WriteTransaction writeConfigTxn) {
1647
1648         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1649                                                 LogicalDatastoreType.OPERATIONAL, identifier);
1650
1651         if (optVpnInterface.isPresent()) {
1652             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1653             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1654             String vpnName = currVpnIntf.getVpnInstanceName();
1655             VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1656             InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1657             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker,
1658                                             LogicalDatastoreType.OPERATIONAL, adjPath);
1659             boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1660             VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1661             long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1662             VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1663             List<Adjacency> adjacencies;
1664             if (optAdjacencies.isPresent()) {
1665                 adjacencies = optAdjacencies.get().getAdjacency();
1666             } else {
1667                 // This code will be hit in case of first PNF adjacency
1668                 adjacencies = new ArrayList<>();
1669             }
1670             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1671             L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1672                     .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1673             Adjacency operationalAdjacency = null;
1674             //Handling dual stack neutron port primary adjacency
1675             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1676                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1677                         currVpnIntf.getName(), vpnName);
1678                 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1679                         currVpnIntf.getName());
1680                 if (interfaceState != null) {
1681                     processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1682                             currVpnIntf.getName(),
1683                             vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1684                 }
1685             }
1686             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1687                     && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1688                 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1689                         : RouteOrigin.STATIC;
1690                 String nh = adj.getNextHopIpList().get(0);
1691                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1692                 synchronized (vpnPrefixKey.intern()) {
1693                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1694                             dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1695                     if (rdToAllocate.isPresent()) {
1696                         input.setRd(rdToAllocate.get());
1697                         operationalAdjacency = populator.createOperationalAdjacency(input);
1698                         int label = operationalAdjacency.getLabel().intValue();
1699                         vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1700                                 currVpnIntf.getVpnInstanceName(), l3vni, origin,
1701                                 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1702                         LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1703                                 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1704                                 vpnName, label, currVpnIntf.getName(), dpnId);
1705                     } else {
1706                         LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName,
1707                                 prefix);
1708                         return;
1709                     }
1710                     // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1711                     // Keeping the MPLS check for now.
1712                     if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1713                         final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1714                         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1715                                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1716                         vpnsToImportRoute.forEach(vpn -> {
1717                             if (vpn.getVrfId() != null) {
1718                                 VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1719                                         dataBroker, vpn.getVpnId(), vpnId, prefix,
1720                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1721                                         .ifPresent(
1722                                             rds -> vpnManager.addExtraRoute(
1723                                                     VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1724                                                     adj.getIpAddress(), nh, rds,
1725                                                     currVpnIntf.getVpnInstanceName(),
1726                                                     l3vni, RouteOrigin.SELF_IMPORTED,
1727                                                     currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn));
1728                             }
1729                         });
1730                     }
1731                 }
1732             } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1733                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1734                         currVpnIntf.getName(), vpnName);
1735
1736                 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1737
1738                 writeOperTxn.merge(
1739                         LogicalDatastoreType.OPERATIONAL,
1740                         VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1741                                 adj.getSubnetId().getValue()), prefix),
1742                         VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1743                                 adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1744
1745                 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1746                         adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, 0 /*l3vni*/,
1747                       null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1748
1749                 input.setRd(adj.getVrfId());
1750             }
1751             if (operationalAdjacency == null) {
1752                 operationalAdjacency = populator.createOperationalAdjacency(input);
1753             }
1754             adjacencies.add(operationalAdjacency);
1755             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1756             VpnInterfaceOpDataEntry newVpnIntf =
1757                 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1758                     aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1759                         currVpnIntf.getGatewayMacAddress());
1760
1761             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1762         }
1763     }
1764
1765     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1766         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1767         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1768     }
1769
1770     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1771             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1772         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1773                                           LogicalDatastoreType.OPERATIONAL, identifier);
1774
1775         if (optVpnInterface.isPresent()) {
1776             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1777
1778             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1779             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1780             if (optAdjacencies.isPresent()) {
1781                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1782
1783                 if (!adjacencies.isEmpty()) {
1784                     LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1785                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1786                     while (adjIt.hasNext()) {
1787                         Adjacency adjElem = adjIt.next();
1788                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1789                             String rd = adjElem.getVrfId();
1790                             adjIt.remove();
1791
1792                             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1793                             VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1794                                     .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1795                                     currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove(),
1796                                             currVpnIntf.getLportTag(), currVpnIntf.getGatewayMacAddress());
1797
1798                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1799                             if (adj.getNextHopIpList() != null) {
1800                                 for (String nh : adj.getNextHopIpList()) {
1801                                     deleteExtraRouteFromCurrentAndImportingVpns(
1802                                         currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1803                                         currVpnIntf.getName(), writeConfigTxn);
1804                                 }
1805                             } else if (adj.isPhysNetworkFunc()) {
1806                                 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1807                                         adj.getIpAddress(), adj.getSubnetId());
1808                                 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1809                                         writeConfigTxn);
1810                             }
1811                             break;
1812                         }
1813
1814                     }
1815                 }
1816                 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1817                         dpnId, adj.getVrfId());
1818             } else {
1819                 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1820                         + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1821             }
1822         }
1823     }
1824
1825     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1826         String rd, String intfName, WriteTransaction writeConfigTxn) {
1827         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1828         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1829         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1830             String vpnRd = vpn.getVrfId();
1831             if (vpnRd != null) {
1832                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1833             }
1834         }
1835     }
1836
1837     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1838         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1839             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1840             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1841     }
1842
1843     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1844         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1845             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1846     }
1847
1848     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1849         WriteTransaction writeOperTxn) {
1850         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1851         if (dpId.equals(BigInteger.ZERO)) {
1852             LOG.error("addToNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1853                     + " association model", vpnInterfaceName, routerName);
1854
1855             return;
1856         }
1857         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1858
1859         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1860             .OPERATIONAL, routerDpnListIdentifier);
1861         RouterInterfaces routerInterface =
1862             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1863                 vpnInterfaceName).build();
1864         if (optionalRouterDpnList.isPresent()) {
1865             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1866                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1867         } else {
1868             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1869             builder.setRouterId(routerName);
1870             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1871             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
1872             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1873                 getRouterId(routerName),
1874                 builder.build(), true);
1875         }
1876     }
1877
1878     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1879         WriteTransaction writeOperTxn) {
1880         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1881         if (dpId.equals(BigInteger.ZERO)) {
1882             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1883                     + " dissociation model", vpnInterfaceName, routerName);
1884
1885             return;
1886         }
1887         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1888         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1889             .OPERATIONAL, routerDpnListIdentifier);
1890         if (optionalRouterDpnList.isPresent()) {
1891             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1892             RouterInterfaces routerInterface =
1893                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1894                     vpnInterfaceName).build();
1895
1896             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1897                 if (routerInterfaces.isEmpty()) {
1898                     if (writeOperTxn != null) {
1899                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1900                     } else {
1901                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1902                     }
1903                 } else {
1904                     if (writeOperTxn != null) {
1905                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1906                             RouterInterfaces.class,
1907                             new RouterInterfacesKey(vpnInterfaceName)));
1908                     } else {
1909                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1910                             routerDpnListIdentifier.child(
1911                                 RouterInterfaces.class,
1912                                 new RouterInterfacesKey(vpnInterfaceName)));
1913                     }
1914                 }
1915             }
1916         }
1917     }
1918
1919     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1920         WriteTransaction writeOperTxn) {
1921         if (dpId.equals(BigInteger.ZERO)) {
1922             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1923                     + " dissociation model", vpnInterfaceName, routerName);
1924
1925             return;
1926         }
1927         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1928         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1929             .OPERATIONAL, routerDpnListIdentifier);
1930         if (optionalRouterDpnList.isPresent()) {
1931             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1932             RouterInterfaces routerInterface =
1933                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1934                     vpnInterfaceName).build();
1935             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1936                 if (routerInterfaces.isEmpty()) {
1937                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1938                 } else {
1939                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1940                         RouterInterfaces.class,
1941                         new RouterInterfacesKey(vpnInterfaceName)));
1942                 }
1943             }
1944         }
1945     }
1946
1947     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1948                                                     WriteTransaction writeConfigTxn, String vpnName) {
1949         if (vpnInterface == null) {
1950             return;
1951         }
1952         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1953         if (adjs == null) {
1954             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1955                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1956             return;
1957         }
1958         for (Adjacency adj : adjs) {
1959             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1960                 String primaryInterfaceIp = adj.getIpAddress();
1961                 String macAddress = adj.getMacAddress();
1962                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1963
1964                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1965                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1966
1967                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1968                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1969                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1970                         routerInt, label, writeConfigTxn);
1971                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1972                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1973                         macAddress);
1974                 return;
1975             }
1976         }
1977         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1978                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1979                 primaryRd, vpnName);
1980     }
1981
1982     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1983                                                     WriteTransaction writeConfigTxn, String vpnName) {
1984         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
1985         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1986         if (adjs != null) {
1987             List<Adjacency> adjsList = adjs.getAdjacency();
1988             for (Adjacency adj : adjsList) {
1989                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1990                     String primaryInterfaceIp = adj.getIpAddress();
1991                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1992                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1993                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1994                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1995                     return;
1996                 }
1997             }
1998         } else {
1999             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2000                     vpnInterface.getName(), rd);
2001         }
2002     }
2003
2004     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2005         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
2006             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
2007                   vpnName, intefaceData.vpnInterface.getName());
2008             return;
2009         }
2010         final VpnInterfaceKey key = intefaceData.identifier
2011                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
2012         final String interfaceName = key.getName();
2013         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2014                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2015         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2016                   intefaceData.identifier, vpnName);
2017         return;
2018     }
2019
2020     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2021                                               VpnInterface vpnInterface, String vpnName) {
2022         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2023                .get(vpnName);
2024         if (vpnInterfaces == null) {
2025             vpnInterfaces = new ConcurrentLinkedQueue<>();
2026         }
2027         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2028         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2029         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2030                 vpnInterface.getName(), vpnName);
2031     }
2032
2033     public boolean isVpnInstanceReady(String vpnInstanceName) {
2034         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2035         if (vpnRd == null) {
2036             return false;
2037         }
2038         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2039
2040         return vpnInstanceOpDataEntry != null;
2041     }
2042
2043     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2044         synchronized (vpnInstanceName.intern()) {
2045             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2046                     unprocessedVpnInterfaces.get(vpnInstanceName);
2047             if (vpnInterfaces != null) {
2048                 while (!vpnInterfaces.isEmpty()) {
2049                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2050                     if (hasVpnInstanceCreatedSuccessfully) {
2051                         processSavedInterface(savedInterface, vpnInstanceName);
2052                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2053                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2054                     } else {
2055                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2056                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2057                     }
2058                 }
2059             } else {
2060                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2061             }
2062         }
2063     }
2064
2065     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2066             VpnInterface vpnInterface) {
2067         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2068             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2069                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2070             if (vpnInterfaces != null) {
2071                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2072                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2073                             + "unprocessed list", vpnInterface.getName(),
2074                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2075                 }
2076             } else {
2077                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2078                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2079             }
2080         }
2081     }
2082
2083     public void vpnInstanceIsReady(String vpnInstanceName) {
2084         processSavedInterfaces(vpnInstanceName, true);
2085     }
2086
2087     public void vpnInstanceFailed(String vpnInstanceName) {
2088         processSavedInterfaces(vpnInstanceName, false);
2089     }
2090
2091     private static class UnprocessedVpnInterfaceData {
2092         InstanceIdentifier<VpnInterface> identifier;
2093         VpnInterface vpnInterface;
2094
2095         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2096             this.identifier = identifier;
2097             this.vpnInterface = vpnInterface;
2098         }
2099
2100         @Override
2101         public int hashCode() {
2102             final int prime = 31;
2103             int result = 1;
2104             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2105             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2106             return result;
2107         }
2108
2109         @Override
2110         public boolean equals(Object obj) {
2111             if (this == obj) {
2112                 return true;
2113             }
2114             if (obj == null) {
2115                 return false;
2116             }
2117             if (getClass() != obj.getClass()) {
2118                 return false;
2119             }
2120             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2121             if (identifier == null) {
2122                 if (other.identifier != null) {
2123                     return false;
2124                 }
2125             } else if (!identifier.equals(other.identifier)) {
2126                 return false;
2127             }
2128             if (vpnInterface == null) {
2129                 if (other.vpnInterface != null) {
2130                     return false;
2131                 }
2132             } else if (!vpnInterface.equals(other.vpnInterface)) {
2133                 return false;
2134             }
2135             return true;
2136         }
2137     }
2138
2139     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2140         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2141         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2142         if (vpnInstanceOpData == null) {
2143             return;
2144         }
2145         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2146         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2147             return;
2148         }
2149         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2150         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2151             InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2152                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2153             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = VpnUtil.read(dataBroker,
2154                     LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2155             if (!vpnInterfaceOptional.isPresent()) {
2156                 return;
2157             }
2158             List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2159                     vpnInterface.getInterfaceName());
2160             if (configVpnAdjacencies == null) {
2161                 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2162                 return;
2163             }
2164             List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2165                     .getAugmentation(AdjacenciesOp.class).getAdjacency();
2166             // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2167             // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2168             // in operational DS. These unprocessed adjacencies will be handled below.
2169             // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2170             // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2171             configVpnAdjacencies.stream()
2172                     .filter(adjacency -> operationVpnAdjacencies.stream()
2173                             .noneMatch(operationalAdjacency ->
2174                                     operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2175                     .forEach(adjacency -> {
2176                         LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2177                         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2178                             () -> {
2179                                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
2180                                 if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2181                                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2182                                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx -> futures.add(
2183                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(
2184                                                 confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd,
2185                                                         adjacency, vpnInterfaceOptional.get().getDpnId(), confTx,
2186                                                         operTx)))));
2187                                     return futures;
2188                                 } else {
2189                                     return Collections.emptyList();
2190                                 }
2191                             });
2192                     });
2193         }));
2194     }
2195
2196     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2197         private final String interfaceName;
2198         private final boolean add;
2199         private final String txnDestination;
2200
2201         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2202             this.interfaceName = interfaceName;
2203             this.add = add;
2204             this.txnDestination = transactionDest;
2205         }
2206
2207         @Override
2208         public void onSuccess(Void voidObj) {
2209             if (add) {
2210                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2211                         interfaceName, txnDestination);
2212             } else {
2213                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2214             }
2215         }
2216
2217         @Override
2218         public void onFailure(Throwable throwable) {
2219             if (add) {
2220                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2221                         interfaceName, txnDestination, throwable);
2222             } else {
2223                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2224                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
2225             }
2226         }
2227     }
2228 }