vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.concurrent.ConcurrentHashMap;
  25 import java.util.concurrent.ConcurrentLinkedQueue;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  42 import org.opendaylight.genius.mdsalutil.NWUtil;
  43 import org.opendaylight.genius.mdsalutil.NwConstants;
  44 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.infrautils.caches.CacheProvider;
  47 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  51 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  52 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  53 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  54 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  55 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  56 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  57 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  58 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  59 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 109 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 110 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 111 import org.slf4j.Logger;
 112 import org.slf4j.LoggerFactory;
 113
 114 @Singleton
 115 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 116
 117     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 118     private static final short DJC_MAX_RETRIES = 3;
 119
 120     private final DataBroker dataBroker;
 121     private final ManagedNewTransactionRunner txRunner;
 122     private final IBgpManager bgpManager;
 123     private final IFibManager fibManager;
 124     private final IMdsalApiManager mdsalManager;
 125     private final IdManagerService idManager;
 126     private final OdlInterfaceRpcService ifaceMgrRpcService;
 127     private final VpnFootprintService vpnFootprintService;
 128     private final IInterfaceManager interfaceManager;
 129     private final IVpnManager vpnManager;
 130     private final ArpResponderHandler arpResponderHandler;
 131     private final JobCoordinator jobCoordinator;
 132
 133     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 134
 135     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 136             new ConcurrentHashMap<>();
 137
 138     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 139
 140     @Inject
 141     public VpnInterfaceManager(final DataBroker dataBroker,
 142                                final IBgpManager bgpManager,
 143                                final IdManagerService idManager,
 144                                final IMdsalApiManager mdsalManager,
 145                                final IFibManager fibManager,
 146                                final OdlInterfaceRpcService ifaceMgrRpcService,
 147                                final VpnFootprintService vpnFootprintService,
 148                                final IInterfaceManager interfaceManager,
 149                                final IVpnManager vpnManager,
 150                                final ArpResponderHandler arpResponderHandler,
 151                                final JobCoordinator jobCoordinator,
 152                                final CacheProvider cacheProvider) {
 153         super(VpnInterface.class, VpnInterfaceManager.class);
 154
 155         this.dataBroker = dataBroker;
 156         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 157         this.bgpManager = bgpManager;
 158         this.idManager = idManager;
 159         this.mdsalManager = mdsalManager;
 160         this.fibManager = fibManager;
 161         this.ifaceMgrRpcService = ifaceMgrRpcService;
 162         this.vpnFootprintService = vpnFootprintService;
 163         this.interfaceManager = interfaceManager;
 164         this.vpnManager = vpnManager;
 165         this.arpResponderHandler = arpResponderHandler;
 166         this.jobCoordinator = jobCoordinator;
 167
 168         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 169                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 170                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 171     }
 172
 173     public Runnable isNotifyTaskQueued(String intfName) {
 174         return vpnIntfMap.remove(intfName);
 175     }
 176
 177     @PostConstruct
 178     public void start() {
 179         LOG.info("{} start", getClass().getSimpleName());
 180         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 181     }
 182
 183     @Override
 184     @PreDestroy
 185     public void close() {
 186         super.close();
 187         vpnInstanceOpDataEntryCache.close();
 188     }
 189
 190     @Override
 191     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 192         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 193     }
 194
 195     @Override
 196     protected VpnInterfaceManager getDataTreeChangeListener() {
 197         return VpnInterfaceManager.this;
 198     }
 199
 200     @Override
 201     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 202         LOG.info("add: intfName {} onto vpnName {}",
 203                  vpnInterface.getName(),
 204                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 205         addVpnInterface(identifier, vpnInterface, null, null);
 206     }
 207
 208     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 209                           final VpnInterface vpnInterface, String vpnName) {
 210         synchronized (vpnName.intern()) {
 211             if (isVpnInstanceReady(vpnName)) {
 212                 return true;
 213             }
 214             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 215             return false;
 216         }
 217     }
 218
 219     // TODO Clean up the exception handling
 220     @SuppressWarnings("checkstyle:IllegalCatch")
 221     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 222                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 223         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 224             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 225             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 226         }
 227     }
 228
 229     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 230                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 231         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 232         final String interfaceName = key.getName();
 233
 234         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 235             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 236                   vpnName, vpnInterface.getName());
 237             return;
 238         }
 239         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 240              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 241         List<Adjacency> copyOldAdjs = null;
 242         if (oldAdjs != null) {
 243             copyOldAdjs = new ArrayList<>();
 244             copyOldAdjs.addAll(oldAdjs);
 245         }
 246         List<Adjacency> copyNewAdjs = null;
 247         if (newAdjs != null) {
 248             copyNewAdjs = new ArrayList<>();
 249             copyNewAdjs.addAll(newAdjs);
 250         }
 251         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 252     }
 253
 254     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 255                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 256                     final List<Adjacency> newAdjs,
 257                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 258         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 259         final String interfaceName = key.getName();
 260         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 261         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 262             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 263             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 264             if (interfaceState != null) {
 265                 try {
 266                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 267                     final int ifIndex = interfaceState.getIfIndex();
 268                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 269                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 270                         WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 271                         WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 272                         LOG.info("addVpnInterface: VPN Interface add event - intfName {} vpnName {} on dpn {}" ,
 273                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 274                         processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, writeConfigTxn,
 275                                      writeOperTxn, writeInvTxn, interfaceState, vpnName);
 276                         if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 277                             LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 278                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 279                             if (newAdjs != null) {
 280                                 for (Adjacency adj : newAdjs) {
 281                                     if (oldAdjs.contains(adj)) {
 282                                         oldAdjs.remove(adj);
 283                                     } else {
 284                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 285                                                                   dataBroker, adj)) {
 286                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd,
 287                                                   adj, dpnId, writeOperTxn, writeConfigTxn);
 288                                         }
 289                                     }
 290                                 }
 291                             }
 292                             for (Adjacency adj : oldAdjs) {
 293                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 294                                                           dataBroker, adj)) {
 295                                     delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 296                                         writeOperTxn, writeConfigTxn);
 297                                 }
 298                             }
 299                         }
 300                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
 301                         try {
 302                             operFuture.get();
 303                         } catch (ExecutionException e) {
 304                             LOG.error("addVpnInterface: Exception encountered while submitting operational future for"
 305                                     + " addVpnInterface {} on vpn {}", vpnInterface.getName(), vpnName, e);
 306                             return null;
 307                         }
 308                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 309                         ListenableFuture<Void> configFuture = writeConfigTxn.submit();
 310                         futures.add(configFuture);
 311                         Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"));
 312                         futures.add(writeInvTxn.submit());
 313                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 314                                 + " processed successfully", interfaceName, vpnName, dpnId);
 315                         return futures;
 316                     });
 317                 } catch (NumberFormatException | IllegalStateException e) {
 318                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 319                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 320                                     vpnName, e);
 321                     return;
 322                 }
 323             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 324                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 325                     () -> {
 326                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 327                         createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 328                                        writeConfigTxn, vpnName);
 329                         LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 330                                  vpnName, vpnInterface.getDpnId());
 331                         ListenableFuture<Void> futures = writeConfigTxn.submit();
 332                         String errorText = "addVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
 333                              + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
 334                         ListenableFutures.addErrorLogging(futures, LOG, errorText);
 335                         return Collections.singletonList(futures);
 336                     });
 337             } else {
 338                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 339                         + " is not available", interfaceName, vpnName);
 340             }
 341         } else {
 342             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 343                       + " as vpn is pending delete", interfaceName, vpnName,
 344                     vpnInterface.getDpnId());
 345         }
 346     }
 347
 348     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 349     // see comments below.
 350     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 351     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 352             final int lportTag, boolean isInterfaceUp,
 353             WriteTransaction writeConfigTxn,
 354             WriteTransaction writeOperTxn,
 355             WriteTransaction writeInvTxn,
 356             Interface interfaceState,
 357             final String vpnName) {
 358         final String interfaceName = vpnInterface.getName();
 359         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 360                                                          interfaceName, vpnName);
 361         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 362         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 363         if (!isInterfaceUp) {
 364             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 365                      interfaceName, dpId, vpnName);
 366             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 367             if (vpnId == VpnConstants.INVALID_ID) {
 368                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 369                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 370                         dpId);
 371                 return;
 372             }
 373
 374             boolean waitForVpnInterfaceOpRemoval = false;
 375             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 376                 String opVpnName = opVpnInterface.getVpnInstanceName();
 377                 String primaryInterfaceIp = null;
 378                 if (opVpnName.equals(vpnName)) {
 379                     // Please check if the primary VRF Entry does not exist for VPNInterface
 380                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 381                     // back to back
 382                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 383                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 384                     if (adjs == null) {
 385                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 386                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 387                                 vpnName);
 388                         return;
 389                     }
 390                     for (Adjacency adj : adjs) {
 391                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 392                             primaryInterfaceIp = adj.getIpAddress();
 393                             break;
 394                         }
 395                     }
 396                     if (primaryInterfaceIp == null) {
 397                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 398                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 399                                 dpId, vpnName);
 400                         return;
 401                     }
 402                     // Get the rd of the vpn instance
 403                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 404                     if (vrf != null) {
 405                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 406                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 407                         return;
 408                     }
 409                     waitForVpnInterfaceOpRemoval = true;
 410                 } else {
 411                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 412                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 413                 }
 414             }
 415             if (!waitForVpnInterfaceOpRemoval) {
 416                 // Add the VPNInterface and quit
 417                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 418                         null/*ipAddressSourceValuePair*/,
 419                         true /* add */);
 420                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 421                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 422                 if (!isBgpVpnInternetVpn) {
 423                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 424                             jobCoordinator);
 425                 }
 426                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 427                         dpId, vpnName);
 428                 if (interfaceManager.isExternalInterface(interfaceName)) {
 429                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 430                         NwConstants.ADD_FLOW);
 431                 }
 432                 return;
 433             }
 434
 435             // FIB didn't get a chance yet to clean up this VPNInterface
 436             // Let us give it a chance here !
 437             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 438                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 439             try {
 440                 Runnable notifyTask = new VpnNotifyTask();
 441                 synchronized (notifyTask) {
 442                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 443                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 444                     // published here while holding the lock on it so this path will hit the wait before notify can be
 445                     // invoked.
 446                     vpnIntfMap.put(interfaceName, notifyTask);
 447                     try {
 448                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 449                     } catch (InterruptedException e) {
 450                         // Ignored
 451                     }
 452                 }
 453             } finally {
 454                 vpnIntfMap.remove(interfaceName);
 455             }
 456
 457             if (opVpnInterface != null) {
 458                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 459                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 460                         dpId, vpnName);
 461                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
 462                 return;
 463             }
 464             // VPNInterface got removed, proceed with Add
 465             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 466                     interfaceName, dpId, vpnName);
 467             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 468                     null/*ipAddressSourceValuePair*/,
 469                     true /* add */);
 470             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 471                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 472             if (!isBgpVpnInternetVpn) {
 473                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 474             }
 475             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 476                     + " FIB to clean up", interfaceName, dpId, vpnName);
 477             if (interfaceManager.isExternalInterface(interfaceName)) {
 478                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 479                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 480             }
 481         } else {
 482             // Interface is retained in the DPN, but its Link Up.
 483             // Advertise prefixes again for this interface to BGP
 484             InstanceIdentifier<VpnInterface> identifier =
 485                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 486             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 487                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 488             advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 489             // Perform similar operation as interface add event for extraroutes.
 490             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 491             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 492             if (!optAdjacencies.isPresent()) {
 493                 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 494                 return;
 495             }
 496             List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 497             for (Adjacency adjacency : adjacencies) {
 498                 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 499                     continue;
 500                 }
 501                 // if BGPVPN Internet, filter only IPv6 Adjacencies
 502                 if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 503                                 dataBroker, adjacency)) {
 504                     continue;
 505                 }
 506                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 507                           dpId, writeOperTxn, writeConfigTxn);
 508             }
 509         }
 510     }
 511
 512     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 513             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 514         Uuid extNetworkId;
 515         try {
 516             // vpn instance of ext-net interface is the network-id
 517             extNetworkId = new Uuid(vpnName);
 518         } catch (IllegalArgumentException e) {
 519             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 520                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 521             return;
 522         }
 523
 524         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 525         if (routerIds == null || routerIds.isEmpty()) {
 526             LOG.info("processExternalVpnInterface: No router is associated with {}."
 527                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 528                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 529             return;
 530         }
 531
 532         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 533                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 534         for (Uuid routerId : routerIds) {
 535             String routerName = routerId.getValue();
 536             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 537             if (Objects.equals(primarySwitch, dpId)) {
 538                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 539                 if (router != null) {
 540                     if (addOrRemove == NwConstants.ADD_FLOW) {
 541                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 542                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 543                                 dpId, interfaceName, lportTag, writeInvTxn);
 544                     } else {
 545                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 546                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 547                                 dpId, interfaceName, lportTag);
 548                     }
 549                 } else {
 550                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 551                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 552                             interfaceName, dpId, vpnName);
 553                 }
 554             }
 555         }
 556     }
 557
 558     // TODO Clean up the exception handling
 559     @SuppressWarnings("checkstyle:IllegalCatch")
 560     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 561                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 562                                                   String vpnName, String interfaceName) {
 563         if (rd == null) {
 564             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 565                   interfaceName, dpnId, vpnName);
 566             return;
 567         } else {
 568             if (rd.equals(vpnName)) {
 569                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 570                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 571                 return;
 572             }
 573         }
 574         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 575                 interfaceName, dpnId, vpnName, rd);
 576
 577         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 578         if (nextHopIp == null) {
 579             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 580                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 581                     rd, vpnName);
 582             return;
 583         }
 584
 585         //Read NextHops
 586         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 587         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 588         if (adjacencies.isPresent()) {
 589             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 590
 591             if (!nextHops.isEmpty()) {
 592                 LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 593                         + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 594                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 595                 long l3vni = vpnInstanceOpData.getL3vni();
 596                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 597                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 598                 for (Adjacency nextHop : nextHops) {
 599                     if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 600                         continue;
 601                     }
 602                     String gatewayMac = null;
 603                     long label = 0;
 604                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 605                         final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 606                             vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 607                         gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName).get();
 608                     } else {
 609                         label = nextHop.getLabel();
 610                     }
 611                     try {
 612                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 613                                 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 614                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 615                                 encapType, (int)label, l3vni, 0 /*l2vni*/,
 616                                 gatewayMac);
 617                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 618                                 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 619                                 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 620                     } catch (Exception e) {
 621                         LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}"
 622                                 + " for interface {} on dpn {}", nextHop.getIpAddress(),
 623                                   vpnName, rd, interfaceName, dpnId, e);
 624                     }
 625                 }
 626             }
 627         }
 628     }
 629
 630     // TODO Clean up the exception handling
 631     @SuppressWarnings("checkstyle:IllegalCatch")
 632     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 633                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 634         //Read NextHops
 635         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 636         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 637
 638         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 639         if (rd == null) {
 640             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 641                 interfaceName, vpnName);
 642             return;
 643         } else {
 644             if (rd.equals(vpnName)) {
 645                 LOG.info(
 646                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 647                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 648                 return;
 649             }
 650         }
 651         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 652                vpnName, rd);
 653         if (adjacencies.isPresent()) {
 654             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 655
 656             if (!nextHops.isEmpty()) {
 657                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 658                         nextHops, interfaceName, vpnName, rd);
 659                 for (Adjacency nextHop : nextHops) {
 660                     try {
 661                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 662                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 663                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 664                                     interfaceName, vpnName);
 665                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 666                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 667                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 668                                     interfaceName, vpnName);
 669                         } else {
 670                             // Perform similar operation as interface delete event for extraroutes.
 671                             String allocatedRd = nextHop.getVrfId();
 672                             for (String nh : nextHop.getNextHopIpList()) {
 673                                 deleteExtraRouteFromCurrentAndImportingVpns(
 674                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 675                             }
 676                         }
 677                     } catch (Exception e) {
 678                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 679                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 680                     }
 681                 }
 682             }
 683         }
 684     }
 685
 686     @SuppressWarnings("checkstyle:IllegalCatch")
 687     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 688                                                   String primaryRd, String interfaceName, final long vpnId,
 689                                                   WriteTransaction writeConfigTxn,
 690                                                   WriteTransaction writeOperTxn,
 691                                                   final WriteTransaction writeInvTxn,
 692                                                   Interface interfaceState) {
 693         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 694         // Read NextHops
 695         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 696         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 697         if (!adjacencies.isPresent()) {
 698             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 699                     null/*gwMac*/, writeOperTxn);
 700             return;
 701         }
 702
 703         // Get the rd of the vpn instance
 704         String nextHopIp = null;
 705         try {
 706             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 707         } catch (Exception e) {
 708             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 709                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 710         }
 711         List<String> nhList = new ArrayList<>();
 712         if (nextHopIp != null) {
 713             nhList.add(nextHopIp);
 714             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 715                     interfaceName, dpnId, vpnName, nhList);
 716         }
 717         Optional<String> gwMac = Optional.absent();
 718         String vpnInterfaceSubnetGwMacAddress = null;
 719         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 720         Long l3vni = vpnInstanceOpData.getL3vni();
 721         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 722         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 723         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 724         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 725         List<Adjacency> value = new ArrayList<>();
 726         for (Adjacency nextHop : nextHops) {
 727             String rd = primaryRd;
 728             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 729             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 730                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 731                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 732                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 733                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 734                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 735                 continue;
 736             }
 737             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 738                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 739                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 740                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 741                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 742                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 743                 writeOperTxn.merge(
 744                     LogicalDatastoreType.OPERATIONAL,
 745                     VpnUtil.getPrefixToInterfaceIdentifier(
 746                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 747                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 748                             prefixCue), true);
 749                 final Uuid subnetId = nextHop.getSubnetId();
 750
 751                 String gatewayIp = nextHop.getSubnetGatewayIp();
 752                 if (gatewayIp == null) {
 753                     Optional<String> gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 754                     if (gatewayIpOptional.isPresent()) {
 755                         gatewayIp = gatewayIpOptional.get();
 756                     }
 757                 }
 758
 759                 if (gatewayIp != null) {
 760                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 761                     if (gwMac.isPresent()) {
 762                         // A valid mac-address is available for this subnet-gateway-ip
 763                         // Use this for programming ARP_RESPONDER table here.  And save this
 764                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 765                         // to populate L3_GW_MAC_TABLE there.
 766                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 767                                 gatewayIp, gwMac.get());
 768                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 769                     } else {
 770                         // A valid mac-address is not available for this subnet-gateway-ip
 771                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 772                         // Save this connected-mac-address as gateway-mac-address for the
 773                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 774                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 775                         if (gwMac.isPresent()) {
 776                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 777                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 778                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 779                                     gatewayIp, gwMac.get());
 780                         } else {
 781                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 782                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 783                                 + "gwIp {}",
 784                                 subnetId, interfaceName, vpnName, gatewayIp);
 785                         }
 786                     }
 787                 } else {
 788                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 789                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 790                         subnetId, interfaceName, vpnName);
 791                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 792                 }
 793                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 794                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 795             } else {
 796                 //Extra route adjacency
 797                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 798                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 799                 synchronized (vpnPrefixKey.intern()) {
 800                     java.util.Optional<String> rdToAllocate = VpnUtil
 801                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 802                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 803                     if (rdToAllocate.isPresent()) {
 804                         rd = rdToAllocate.get();
 805                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 806                             rd, prefix);
 807                     } else {
 808                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 809                         continue;
 810                     }
 811                 }
 812                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 813                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 814                         interfaceName, dpnId);
 815             }
 816             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 817             // can be different from the gateway-mac-address within the VRFEntry as the
 818             // gateway-mac-address is a superset.
 819             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 820                     : RouteOrigin.STATIC;
 821             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 822                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 823                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 824             Adjacency operationalAdjacency = null;
 825             try {
 826                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 827             } catch (NullPointerException e) {
 828                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 829                     input, e.getMessage());
 830                 return;
 831             }
 832             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 833                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 834                         vpnName, l3vni, origin,
 835                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 836             }
 837             value.add(operationalAdjacency);
 838         }
 839
 840         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 841         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 842                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 843
 844         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 845                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 846                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 847
 848         for (Adjacency nextHop : aug.getAdjacency()) {
 849             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 850             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 851                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 852                         : RouteOrigin.STATIC;
 853                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 854                 registeredPopulator.populateFib(input, writeConfigTxn);
 855             }
 856         }
 857     }
 858
 859     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 860                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 861         VpnInterfaceOpDataEntry opInterface =
 862               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 863         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 864             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 865         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 866                 WriteTransaction.CREATE_MISSING_PARENTS);
 867         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 868                 interfaceName, dpnId, vpnName);
 869     }
 870
 871     // TODO Clean up the exception handling
 872     @SuppressWarnings("checkstyle:IllegalCatch")
 873     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 874                                            StateTunnelList stateTunnelList,
 875                                            WriteTransaction writeConfigTxn,
 876                                            WriteTransaction writeOperTxn) {
 877
 878         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 879         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 880         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 881         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 882         if (adjList.isEmpty()) {
 883             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 884                     vpnInterface, srcDpnId);
 885             return;
 886         }
 887         String prefix = null;
 888         long label = 0;
 889         List<Adjacency> value = new ArrayList<>();
 890         boolean isNextHopAddReqd = false;
 891         String vpnName = vpnInterface.getVpnInstanceName();
 892         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 893         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 894         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 895                 vpnInterface.getName(), vpnInterface.getDpnId(),
 896                 vpnInterface.getVpnInstanceName(), adjList);
 897         for (Adjacency adj : adjList) {
 898             String rd = adj.getVrfId();
 899             rd = rd != null ? rd : vpnName;
 900             prefix = adj.getIpAddress();
 901             label = adj.getLabel();
 902             List<String> nhList = Collections.singletonList(srcTepIp);
 903             List<String> nextHopList = adj.getNextHopIpList();
 904             // If TEP is added , update the nexthop of primary adjacency.
 905             // Secondary adj nexthop is already pointing to primary adj IP address.
 906             if (nextHopList != null && !nextHopList.isEmpty()) {
 907                 /* everything right already */
 908             } else {
 909                 isNextHopAddReqd = true;
 910             }
 911
 912             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 913                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 914             } else {
 915                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 916                 if (!vrfEntryOptional.isPresent()) {
 917                     continue;
 918                 }
 919                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 920                 if (!nhList.contains(srcTepIp)) {
 921                     nhList.add(srcTepIp);
 922                     isNextHopAddReqd = true;
 923                 }
 924                 value.add(adj);
 925             }
 926
 927             if (isNextHopAddReqd) {
 928                 updateLabelMapper(label, nhList);
 929                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 930                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 931                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 932                 // Update the VRF entry with nextHop
 933                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 934                         label, true, writeConfigTxn);
 935
 936                 //Get the list of VPN's importing this route(prefix) .
 937                 // Then update the VRF entry with nhList
 938                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 939                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 940                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 941                     String vpnRd = vpn.getVrfId();
 942                     if (vpnRd != null) {
 943                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 944                                 srcTepIp, label, true, writeConfigTxn);
 945                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 946                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 947                             vpnInterface.getName(), srcDpnId, vpnName,
 948                             vpn.getVpnInstanceName(), vpnRd);
 949                     }
 950                 }
 951                 // Advertise the prefix to BGP only for external vpn
 952                 // since there is a nexthop change.
 953                 try {
 954                     if (!rd.equalsIgnoreCase(vpnName)) {
 955                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 956                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 957                                 null /*gatewayMacAddress*/);
 958                     }
 959                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 960                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 961                             srcDpnId, vpnName);
 962                 } catch (Exception ex) {
 963                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 964                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 965                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 966                 }
 967             }
 968         }
 969         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 970         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 971                 .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 972                 .addAugmentation(AdjacenciesOp.class, aug).build();
 973         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 974                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 975         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 976                 WriteTransaction.CREATE_MISSING_PARENTS);
 977         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 978                 vpnInterface.getName(), srcDpnId, vpnName);
 979
 980     }
 981
 982     // TODO Clean up the exception handling
 983     @SuppressWarnings("checkstyle:IllegalCatch")
 984     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
 985                                               StateTunnelList stateTunnelList,
 986                                               WriteTransaction writeConfigTxn,
 987                                               WriteTransaction writeOperTxn) {
 988
 989         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 990         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 991         String prefix = null;
 992         long label = 0;
 993         boolean isNextHopRemoveReqd = false;
 994         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 995         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 996         String vpnName = vpnInterface.getVpnInstanceName();
 997         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 998         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 999         if (adjList != null) {
1000             List<Adjacency> value = new ArrayList<>();
1001             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1002                     vpnInterface.getName(), vpnInterface.getDpnId(),
1003                     vpnInterface.getVpnInstanceName(), adjList);
1004             for (Adjacency adj : adjList) {
1005                 List<String> nhList = new ArrayList<>();
1006                 String rd = adj.getVrfId();
1007                 rd = rd != null ? rd : vpnName;
1008                 prefix = adj.getIpAddress();
1009                 List<String> nextHopList = adj.getNextHopIpList();
1010                 label = adj.getLabel();
1011                 if (nextHopList != null && !nextHopList.isEmpty()) {
1012                     isNextHopRemoveReqd = true;
1013                 }
1014                 // If TEP is deleted , remove the nexthop from primary adjacency.
1015                 // Secondary adj nexthop will continue to point to primary adj IP address.
1016                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1017                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1018                 } else {
1019                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1020                     if (!vrfEntryOptional.isPresent()) {
1021                         continue;
1022                     }
1023                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1024                     if (nhList.contains(srcTepIp)) {
1025                         nhList.remove(srcTepIp);
1026                         isNextHopRemoveReqd = true;
1027                     }
1028                     value.add(adj);
1029                 }
1030
1031                 if (isNextHopRemoveReqd) {
1032                     updateLabelMapper(label, nhList);
1033                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1034                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1035                             prefix, nhList, vpnName,
1036                             vpnId, rd, vpnInterface.getName());
1037                     // Update the VRF entry with removed nextHop
1038                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1039                             label, false, writeConfigTxn);
1040
1041                     //Get the list of VPN's importing this route(prefix) .
1042                     // Then update the VRF entry with nhList
1043                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1044                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1045                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1046                         String vpnRd = vpn.getVrfId();
1047                         if (vpnRd != null) {
1048                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1049                                     srcTepIp, label, false, writeConfigTxn);
1050                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1051                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1052                                     nhList, label, vpnInterface.getName(), srcDpnId,
1053                                     vpnName,
1054                                     vpn.getVpnInstanceName(), vpnRd);
1055                         }
1056                     }
1057
1058                     // Withdraw prefix from BGP only for external vpn.
1059                     try {
1060                         if (!rd.equalsIgnoreCase(vpnName)) {
1061                             bgpManager.withdrawPrefix(rd, prefix);
1062                         }
1063                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1064                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1065                                 vpnInterface.getName(), srcDpnId,
1066                                 vpnName);
1067                     } catch (Exception ex) {
1068                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1069                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1070                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1071                     }
1072                 }
1073             }
1074             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1075             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1076                     .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1077                     .addAugmentation(AdjacenciesOp.class, aug).build();
1078             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1079                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1080             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1081                     WriteTransaction.CREATE_MISSING_PARENTS);
1082             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1083                          vpnInterface.getName(), srcDpnId, vpnName);
1084         }
1085     }
1086
1087     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1088         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1089
1090         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1091         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1092         if (vpnInstanceOpDataEntry == null) {
1093             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1094                     + " to check for vpns exporting the routes", vpnName);
1095             return vpnsToExportRoute;
1096         }
1097
1098         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1099             if (input.getVpnInstanceName() == null) {
1100                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1101                         input.getVrfId());
1102                 return false;
1103             }
1104             return !input.getVpnInstanceName().equals(vpnName);
1105         };
1106
1107         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1108             Iterable<String> commonRTs =
1109                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1110                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1111             return Iterators.size(commonRTs.iterator()) > 0;
1112         };
1113
1114         vpnsToExportRoute =
1115                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1116                         Collectors.toList());
1117         return vpnsToExportRoute;
1118     }
1119
1120     // TODO Clean up the exception handling
1121     @SuppressWarnings("checkstyle:IllegalCatch")
1122     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1123         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1124         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1125             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1126             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1127             if (vrfEntries != null) {
1128                 for (VrfEntry vrfEntry : vrfEntries) {
1129                     try {
1130                         if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1131                                 RouteOrigin.value(vrfEntry.getOrigin()))) {
1132                             LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1133                                     + " is not a controller managed non intervpn link route. Ignoring.",
1134                                     vpn.getVrfId(), vrfEntry.getDestPrefix());
1135                             continue;
1136                         }
1137                         String prefix = vrfEntry.getDestPrefix();
1138                         String gwMac = vrfEntry.getGatewayMacAddress();
1139                         vrfEntry.getRoutePaths().forEach(routePath -> {
1140                             String nh = routePath.getNexthopAddress();
1141                             int label = routePath.getLabel().intValue();
1142                             if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1143                                     vrfEntry.getOrigin()))) {
1144                                 LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1145                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1146                                         vpnName, vpnRd);
1147                                 fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1148                                         Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1149                                         0 /*l3vni*/, gwMac,  vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1150                                         writeConfigTxn);
1151                             } else {
1152                                 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} prefix {}"
1153                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1154                                         vpnName, vpnRd);
1155                                 SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1156                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1157                                         writeConfigTxn);
1158                             }
1159                         });
1160                     } catch (RuntimeException e) {
1161                         LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1162                                 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1163                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1164                     }
1165                 }
1166                 writeConfigTxn.submit();
1167             } else {
1168                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1169                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1170             }
1171         }
1172     }
1173
1174     @SuppressWarnings("checkstyle:IllegalCatch")
1175     @Override
1176     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1177         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1178         final String interfaceName = key.getName();
1179         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1180             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1181             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1182         }
1183     }
1184
1185     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1186                                 final VpnInterface vpnInterface, final String vpnName,
1187                                 final String interfaceName) {
1188         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1189             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1190                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1191                 deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn, vpnName);
1192                 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1193                 ListenableFuture<Void> futures = writeConfigTxn.submit();
1194                 String errorText = "removeVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
1195                         + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
1196                 ListenableFutures.addErrorLogging(futures, LOG, errorText);
1197                 return Collections.singletonList(futures);
1198             }, DJC_MAX_RETRIES);
1199         } else {
1200             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1201             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1202         }
1203     }
1204
1205     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1206     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1207                                 final VpnInterface vpnInterface, final String vpnName,
1208                                 final String interfaceName, final Interface interfaceState) {
1209         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1210                 vpnName, vpnInterface.getDpnId());
1211         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1212         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1213             () -> {
1214                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1215                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1216                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1217                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1218                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1219                                     interfaceName, vpnName);
1220                             BigInteger dpId = BigInteger.ZERO;
1221                             int ifIndex = 0;
1222                             String gwMacAddress = null;
1223                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1224                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1225                             final Optional<VpnInterfaceOpDataEntry> optVpnInterface =
1226                                     VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1227                             if (interfaceState != null) {
1228                                 try {
1229                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1230                                 } catch (NumberFormatException | IllegalStateException e) {
1231                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1232                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1233                                                     + " from vpn interface op data store. ", interfaceName,
1234                                             vpnInterface.getDpnId(), vpnName, e);
1235                                     dpId = BigInteger.ZERO;
1236                                 }
1237                                 ifIndex = interfaceState.getIfIndex();
1238                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1239                             } else {
1240                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1241                                         + " from vpn interface op.", interfaceName);
1242                                 if (optVpnInterface.isPresent()) {
1243                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1244                                     dpId = vpnOpInterface.getDpnId();
1245                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1246                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1247                                 } else {
1248                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1249                                                     + " as interfaceState and vpn interface op is not"
1250                                             + " available", interfaceName, vpnName);
1251                                     return;
1252                                 }
1253                             }
1254                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1255                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1256                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1257                             LOG.info(
1258                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1259                                             + "successfully",
1260                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1261                         })))));
1262                 futures.add(configFuture);
1263                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1264                 return futures;
1265             }, DJC_MAX_RETRIES);
1266     }
1267
1268     protected void processVpnInterfaceDown(BigInteger dpId,
1269                                            String interfaceName,
1270                                            int lportTag,
1271                                            String gwMac,
1272                                            VpnInterfaceOpDataEntry vpnOpInterface,
1273                                            boolean isInterfaceStateDown,
1274                                            WriteTransaction writeConfigTxn,
1275                                            WriteTransaction writeOperTxn,
1276                                            WriteTransaction writeInvTxn) {
1277         if (vpnOpInterface == null) {
1278             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1279                     + " as it is not available in operational data store", interfaceName, dpId);
1280             return;
1281         }
1282         final String vpnName = vpnOpInterface.getVpnInstanceName();
1283         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1284                                                     interfaceName, vpnName);
1285         if (!isInterfaceStateDown) {
1286             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1287             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1288                     null);
1289             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1290             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1291                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1292             if (interfaceManager.isExternalInterface(interfaceName)) {
1293                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1294                         NwConstants.DEL_FLOW);
1295             }
1296             if (!isBgpVpnInternetVpn) {
1297                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1298             }
1299             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1300                     + " successful", interfaceName, dpId, vpnName);
1301         } else {
1302             // Interface is retained in the DPN, but its Link Down.
1303             // Only withdraw the prefixes for this interface from BGP
1304             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1305         }
1306     }
1307
1308     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1309                                           final String vpnName, final long vpnId, String gwMac,
1310                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1311                                           final WriteTransaction writeInvTxn) {
1312         //Read NextHops
1313         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1314                 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1315         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1316         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1317
1318         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1319         LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1320                 interfaceName, dpnId, vpnName, primaryRd);
1321         if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1322             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1323             LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1324                     interfaceName, dpnId, vpnName, nextHops);
1325             for (Adjacency nextHop : nextHops) {
1326                 if (nextHop.isPhysNetworkFunc()) {
1327                     LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1328                             nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1329                     fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1330                             null/*writeCfgTxn*/);
1331                 } else {
1332                     String rd = nextHop.getVrfId();
1333                     List<String> nhList;
1334                     if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1335                         nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1336                     } else {
1337                         // This is a primary adjacency
1338                         nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1339                                 : Collections.emptyList();
1340                         removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1341                                 interfaceName, writeInvTxn);
1342                     }
1343                     if (!nhList.isEmpty()) {
1344                         if (rd.equals(vpnName)) {
1345                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1346                             //remove from FIB directly
1347                             nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1348                                     interfaceName, dpnId, writeConfigTxn));
1349                         } else {
1350                             removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, interfaceName,
1351                                     writeConfigTxn);
1352                         }
1353                     } else {
1354                         LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1355                                         + " interface {}", nextHop.getIpAddress(), rd,
1356                                 nextHop.getAdjacencyType().toString(), interfaceName);
1357                         bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1358                         fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1359                     }
1360                 }
1361                 String ip = nextHop.getIpAddress().split("/")[0];
1362                 LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1363                 if (vpnVipToPort != null) {
1364                     VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip);
1365                     LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1366                                     + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(),
1367                             ip, dpnId, vpnName);
1368                 }
1369             }
1370         } else {
1371             // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1372             LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1373                     + " Removing it.", interfaceName, vpnName, dpnId);
1374             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1375         }
1376     }
1377
1378     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1379                                                                    String interfaceName, BigInteger dpnId,
1380                                                                    WriteTransaction writeConfigTxn) {
1381         return (nh) -> {
1382             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1383                     writeConfigTxn);
1384             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1385                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1386                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1387         };
1388     }
1389
1390     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1391                                            BigInteger dpnId, String rd, String interfaceName,
1392                                            WriteTransaction writeConfigTxn) {
1393         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1394                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1395         nhList.forEach((nh) -> {
1396             //IRT: remove routes from other vpns importing it
1397             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1398                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1399             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1400                 String vpnRd = vpn.getVrfId();
1401                 if (vpnRd != null) {
1402                     fibManager.removeOrUpdateFibEntry(vpnRd,
1403                             nextHop.getIpAddress(), nh, writeConfigTxn);
1404                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1405                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1406                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1407                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1408                 }
1409             }
1410         });
1411     }
1412
1413     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1414                                                  int lportTag, String gwMac, String interfaceName,
1415                                                  WriteTransaction writeInvTxn) {
1416         final Uuid subnetId = nextHop.getSubnetId();
1417         if (nextHop.getSubnetGatewayMacAddress() == null) {
1418             // A valid mac-address was not available for this subnet-gateway-ip
1419             // So a connected-mac-address was used for this subnet and we need
1420             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1421             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1422                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1423         }
1424         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1425                 subnetId);
1426     }
1427
1428     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1429                                                   String interfaceName) {
1430         // This is either an extra-route (or) a learned IP via subnet-route
1431         List<String> nhList = null;
1432         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1433         if (nextHopIp == null || nextHopIp.isEmpty()) {
1434             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1435                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1436                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1437                     vpnName);
1438             nhList = Collections.emptyList();
1439         } else {
1440             nhList = Collections.singletonList(nextHopIp);
1441         }
1442         return nhList;
1443     }
1444
1445     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1446         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1447         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1448         // else use connected interface
1449         if (gwPort != null && gwPort.isSubnetIp()) {
1450             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1451                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1452             return Optional.of(gwPort.getMacAddress());
1453         }
1454         return Optional.absent();
1455     }
1456
1457     @Override
1458     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1459         final VpnInterface update) {
1460         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1461                 update.getDpnId(), original.getVpnInstanceNames(),
1462                 update.getVpnInstanceNames());
1463         final String vpnInterfaceName = update.getName();
1464         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1465         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1466         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1467             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1468         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1469         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1470             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1471
1472         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1473         //handles switching between <internal VPN - external VPN>
1474         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1475             LOG.info("update: handled VPNInterface {} on dpn {} update"
1476                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1477                      original.getName(), dpnId,
1478                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1479                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1480             return;
1481         }
1482         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1483             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1484             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1485             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1486             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1487             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1488                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1489                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1490                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1491                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1492                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1493                     LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1494                             update.getName(), newVpnName);
1495                     //handle both addition and removal of adjacencies
1496                     //currently, new adjacency may be an extra route
1497                     boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1498                     if (!oldAdjs.equals(newAdjs)) {
1499                         for (Adjacency adj : copyNewAdjs) {
1500                             if (copyOldAdjs.contains(adj)) {
1501                                 copyOldAdjs.remove(adj);
1502                             } else {
1503                                 // add new adjacency - right now only extra route will hit this path
1504                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1505                                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1506                                              dpnId, writeOperTxn, writeConfigTxn);
1507                                 }
1508                                 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to vpn "
1509                                                 + "interface {} on vpn {} dpnId {}",
1510                                         adj.getIpAddress(), adj.getNextHopIpList(),
1511                                         adj.getLabel(), adj.getSubnetId(), update.getName(),
1512                                         newVpnName, dpnId);
1513                             }
1514                         }
1515                         for (Adjacency adj : copyOldAdjs) {
1516                             if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1517                                 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1518                                        writeOperTxn, writeConfigTxn);
1519                             }
1520                             LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1521                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1522                                 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1523                         }
1524                     }
1525                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
1526                     try {
1527                         operFuture.get();
1528                     } catch (ExecutionException e) {
1529                         LOG.error("Exception encountered while submitting operational future for update"
1530                                 + " VpnInterface {} on vpn {}", vpnInterfaceName, newVpnName, e);
1531                         return null;
1532                     }
1533                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1534                     futures.add(writeConfigTxn.submit());
1535                     LOG.info("update: vpn interface updated for interface {} oldVpn(s) {} newVpn {}"
1536                         + "processed successfully", update.getName(),
1537                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), newVpnName);
1538                     return futures;
1539                 });
1540             } else {
1541                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1542                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1543             }
1544         }
1545     }
1546
1547     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1548                                                  VpnInterface original, VpnInterface update) {
1549         boolean isSwap = Boolean.FALSE;
1550         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1551         final String interfaceName = key.getName();
1552         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1553             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1554         List<String> oldVpnListCopy = new ArrayList<>();
1555         oldVpnListCopy.addAll(oldVpnList);
1556         List<String> newVpnList = update.getVpnInstanceNames().stream()
1557             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1558         oldVpnList.removeAll(newVpnList);
1559         newVpnList.removeAll(oldVpnListCopy);
1560         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1561             for (String oldVpnName: oldVpnList) {
1562                 isSwap = Boolean.TRUE;
1563                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1564                         + " running config-driven swap removal", interfaceName, oldVpnName);
1565                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1566                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1567                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1568             }
1569             //Wait for previous interface bindings to be removed
1570             try {
1571                 Thread.sleep(2000);
1572             } catch (InterruptedException e) {
1573                 //Ignore
1574             }
1575             for (String newVpnName: newVpnList) {
1576                 String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1577                 isSwap = Boolean.TRUE;
1578                 if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1579                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1580                             + "running config-driven swap addition", interfaceName, newVpnName);
1581                     final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1582                     final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1583                             ? origAdjs.getAdjacency() : new ArrayList<>();
1584                     final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1585                     final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1586                             ? updateAdjs.getAdjacency() : new ArrayList<>();
1587
1588                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1589                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1590                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1591                             interfaceName, oldVpnListCopy, newVpnName);
1592                 }
1593             }
1594         }
1595         return isSwap;
1596     }
1597
1598     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1599         Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1600         synchronized (label.toString().intern()) {
1601             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1602                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1603             Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1604             if (opResult.isPresent()) {
1605                 LabelRouteInfo labelRouteInfo =
1606                     new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1607                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1608             }
1609         }
1610         LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList);
1611     }
1612
1613     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1614         SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1615
1616         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1617         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1618                 .addAugmentation(SubnetRoute.class, route).build();
1619         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1620         InstanceIdentifierBuilder<VrfTables> idBuilder =
1621             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1622         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1623         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1624         if (writeConfigTxn != null) {
1625             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1626         } else {
1627             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1628         }
1629         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1630                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1631     }
1632
1633     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1634                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1635                                            WriteTransaction writeConfigTxn) {
1636
1637         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1638                                                 LogicalDatastoreType.OPERATIONAL, identifier);
1639
1640         if (optVpnInterface.isPresent()) {
1641             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1642             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1643             String vpnName = currVpnIntf.getVpnInstanceName();
1644             VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1645             InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1646             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker,
1647                                             LogicalDatastoreType.OPERATIONAL, adjPath);
1648             boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1649             VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1650             long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1651             VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1652             List<Adjacency> adjacencies;
1653             if (optAdjacencies.isPresent()) {
1654                 adjacencies = optAdjacencies.get().getAdjacency();
1655             } else {
1656                 // This code will be hit in case of first PNF adjacency
1657                 adjacencies = new ArrayList<>();
1658             }
1659             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1660             L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1661                     .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1662             Adjacency operationalAdjacency = null;
1663             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) {
1664                 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1665                         : RouteOrigin.STATIC;
1666                 String nh = adj.getNextHopIpList().get(0);
1667                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1668                 synchronized (vpnPrefixKey.intern()) {
1669                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1670                             dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1671                     if (rdToAllocate.isPresent()) {
1672                         input.setRd(rdToAllocate.get());
1673                         operationalAdjacency = populator.createOperationalAdjacency(input);
1674                         int label = operationalAdjacency.getLabel().intValue();
1675                         vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1676                                 currVpnIntf.getVpnInstanceName(), l3vni, origin,
1677                                 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1678                         LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1679                                 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1680                                 vpnName, label, currVpnIntf.getName(), dpnId);
1681                     } else {
1682                         LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName,
1683                                 prefix);
1684                         return;
1685                     }
1686                     // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1687                     // Keeping the MPLS check for now.
1688                     if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1689                         final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1690                         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1691                                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1692                         vpnsToImportRoute.forEach(vpn -> {
1693                             if (vpn.getVrfId() != null) {
1694                                 VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1695                                         dataBroker, vpn.getVpnId(), vpnId, prefix,
1696                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1697                                         .ifPresent(
1698                                             rds -> vpnManager.addExtraRoute(
1699                                                     VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1700                                                     adj.getIpAddress(), nh, rds,
1701                                                     currVpnIntf.getVpnInstanceName(),
1702                                                     l3vni, RouteOrigin.SELF_IMPORTED,
1703                                                     currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn));
1704                             }
1705                         });
1706                     }
1707                 }
1708             } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1709                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1710                         currVpnIntf.getName(), vpnName);
1711
1712                 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1713
1714                 writeOperTxn.merge(
1715                         LogicalDatastoreType.OPERATIONAL,
1716                         VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1717                                 adj.getSubnetId().getValue()), prefix),
1718                         VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1719                                 adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1720
1721                 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1722                         adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, 0 /*l3vni*/,
1723                       null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1724
1725                 input.setRd(adj.getVrfId());
1726             }
1727             if (operationalAdjacency == null) {
1728                 operationalAdjacency = populator.createOperationalAdjacency(input);
1729             }
1730             adjacencies.add(operationalAdjacency);
1731             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1732             VpnInterfaceOpDataEntry newVpnIntf =
1733                 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1734                     aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1735                         currVpnIntf.getGatewayMacAddress());
1736
1737             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1738         }
1739     }
1740
1741     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1742         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1743         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1744     }
1745
1746     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1747             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1748         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1749                                           LogicalDatastoreType.OPERATIONAL, identifier);
1750
1751         if (optVpnInterface.isPresent()) {
1752             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1753
1754             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1755             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1756             if (optAdjacencies.isPresent()) {
1757                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1758
1759                 if (!adjacencies.isEmpty()) {
1760                     LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1761                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1762                     while (adjIt.hasNext()) {
1763                         Adjacency adjElem = adjIt.next();
1764                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1765                             String rd = adjElem.getVrfId();
1766                             adjIt.remove();
1767
1768                             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1769                             VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1770                                     .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1771                                     currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove(),
1772                                             currVpnIntf.getLportTag(), currVpnIntf.getGatewayMacAddress());
1773
1774                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1775                             if (adj.getNextHopIpList() != null) {
1776                                 for (String nh : adj.getNextHopIpList()) {
1777                                     deleteExtraRouteFromCurrentAndImportingVpns(
1778                                         currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1779                                         currVpnIntf.getName(), writeConfigTxn);
1780                                 }
1781                             } else if (adj.isPhysNetworkFunc()) {
1782                                 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1783                                         adj.getIpAddress(), adj.getSubnetId());
1784                                 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1785                                         writeConfigTxn);
1786                             }
1787                             break;
1788                         }
1789
1790                     }
1791                 }
1792                 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1793                         dpnId, adj.getVrfId());
1794             } else {
1795                 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1796                         + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1797             }
1798         }
1799     }
1800
1801     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1802         String rd, String intfName, WriteTransaction writeConfigTxn) {
1803         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1804         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1805         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1806             String vpnRd = vpn.getVrfId();
1807             if (vpnRd != null) {
1808                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1809             }
1810         }
1811     }
1812
1813     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1814         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1815             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1816             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1817     }
1818
1819     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1820         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1821             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1822     }
1823
1824     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1825         WriteTransaction writeOperTxn) {
1826         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1827         if (dpId.equals(BigInteger.ZERO)) {
1828             LOG.error("addToNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1829                     + " association model", vpnInterfaceName, routerName);
1830
1831             return;
1832         }
1833         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1834
1835         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1836             .OPERATIONAL, routerDpnListIdentifier);
1837         RouterInterfaces routerInterface =
1838             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1839                 vpnInterfaceName).build();
1840         if (optionalRouterDpnList.isPresent()) {
1841             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1842                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1843         } else {
1844             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1845             builder.setRouterId(routerName);
1846             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1847             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
1848             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1849                 getRouterId(routerName),
1850                 builder.build(), true);
1851         }
1852     }
1853
1854     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1855         WriteTransaction writeOperTxn) {
1856         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1857         if (dpId.equals(BigInteger.ZERO)) {
1858             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1859                     + " dissociation model", vpnInterfaceName, routerName);
1860
1861             return;
1862         }
1863         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1864         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1865             .OPERATIONAL, routerDpnListIdentifier);
1866         if (optionalRouterDpnList.isPresent()) {
1867             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1868             RouterInterfaces routerInterface =
1869                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1870                     vpnInterfaceName).build();
1871
1872             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1873                 if (routerInterfaces.isEmpty()) {
1874                     if (writeOperTxn != null) {
1875                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1876                     } else {
1877                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1878                     }
1879                 } else {
1880                     if (writeOperTxn != null) {
1881                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1882                             RouterInterfaces.class,
1883                             new RouterInterfacesKey(vpnInterfaceName)));
1884                     } else {
1885                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1886                             routerDpnListIdentifier.child(
1887                                 RouterInterfaces.class,
1888                                 new RouterInterfacesKey(vpnInterfaceName)));
1889                     }
1890                 }
1891             }
1892         }
1893     }
1894
1895     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1896         WriteTransaction writeOperTxn) {
1897         if (dpId.equals(BigInteger.ZERO)) {
1898             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1899                     + " dissociation model", vpnInterfaceName, routerName);
1900
1901             return;
1902         }
1903         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1904         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1905             .OPERATIONAL, routerDpnListIdentifier);
1906         if (optionalRouterDpnList.isPresent()) {
1907             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1908             RouterInterfaces routerInterface =
1909                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1910                     vpnInterfaceName).build();
1911             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1912                 if (routerInterfaces.isEmpty()) {
1913                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1914                 } else {
1915                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1916                         RouterInterfaces.class,
1917                         new RouterInterfacesKey(vpnInterfaceName)));
1918                 }
1919             }
1920         }
1921     }
1922
1923     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1924                                                     WriteTransaction writeConfigTxn, String vpnName) {
1925         if (vpnInterface == null) {
1926             return;
1927         }
1928         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1929         if (adjs == null) {
1930             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1931                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1932             return;
1933         }
1934         for (Adjacency adj : adjs) {
1935             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1936                 String primaryInterfaceIp = adj.getIpAddress();
1937                 String macAddress = adj.getMacAddress();
1938                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1939
1940                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1941                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1942
1943                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1944                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1945                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1946                         routerInt, label, writeConfigTxn);
1947                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1948                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1949                         macAddress);
1950                 return;
1951             }
1952         }
1953         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1954                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1955                 primaryRd, vpnName);
1956     }
1957
1958     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1959                                                     WriteTransaction writeConfigTxn, String vpnName) {
1960         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
1961         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1962         if (adjs != null) {
1963             List<Adjacency> adjsList = adjs.getAdjacency();
1964             for (Adjacency adj : adjsList) {
1965                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1966                     String primaryInterfaceIp = adj.getIpAddress();
1967                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1968                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1969                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1970                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1971                     return;
1972                 }
1973             }
1974         } else {
1975             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1976                     vpnInterface.getName(), rd);
1977         }
1978     }
1979
1980     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1981         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1982             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1983                   vpnName, intefaceData.vpnInterface.getName());
1984             return;
1985         }
1986         final VpnInterfaceKey key = intefaceData.identifier
1987                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1988         final String interfaceName = key.getName();
1989         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1990                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1991         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1992                   intefaceData.identifier, vpnName);
1993         return;
1994     }
1995
1996     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1997                                               VpnInterface vpnInterface, String vpnName) {
1998         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
1999                .get(vpnName);
2000         if (vpnInterfaces == null) {
2001             vpnInterfaces = new ConcurrentLinkedQueue<>();
2002         }
2003         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2004         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2005         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2006                 vpnInterface.getName(), vpnName);
2007     }
2008
2009     public boolean isVpnInstanceReady(String vpnInstanceName) {
2010         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2011         if (vpnRd == null) {
2012             return false;
2013         }
2014         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2015
2016         return vpnInstanceOpDataEntry != null;
2017     }
2018
2019     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2020         synchronized (vpnInstanceName.intern()) {
2021             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2022                     unprocessedVpnInterfaces.get(vpnInstanceName);
2023             if (vpnInterfaces != null) {
2024                 while (!vpnInterfaces.isEmpty()) {
2025                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2026                     if (hasVpnInstanceCreatedSuccessfully) {
2027                         processSavedInterface(savedInterface, vpnInstanceName);
2028                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2029                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2030                     } else {
2031                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2032                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2033                     }
2034                 }
2035             } else {
2036                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2037             }
2038         }
2039     }
2040
2041     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2042             VpnInterface vpnInterface) {
2043         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2044             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2045                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2046             if (vpnInterfaces != null) {
2047                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2048                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2049                             + "unprocessed list", vpnInterface.getName(),
2050                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2051                 }
2052             } else {
2053                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2054                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2055             }
2056         }
2057     }
2058
2059     public void vpnInstanceIsReady(String vpnInstanceName) {
2060         processSavedInterfaces(vpnInstanceName, true);
2061     }
2062
2063     public void vpnInstanceFailed(String vpnInstanceName) {
2064         processSavedInterfaces(vpnInstanceName, false);
2065     }
2066
2067     private static class UnprocessedVpnInterfaceData {
2068         InstanceIdentifier<VpnInterface> identifier;
2069         VpnInterface vpnInterface;
2070
2071         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2072             this.identifier = identifier;
2073             this.vpnInterface = vpnInterface;
2074         }
2075
2076         @Override
2077         public int hashCode() {
2078             final int prime = 31;
2079             int result = 1;
2080             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2081             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2082             return result;
2083         }
2084
2085         @Override
2086         public boolean equals(Object obj) {
2087             if (this == obj) {
2088                 return true;
2089             }
2090             if (obj == null) {
2091                 return false;
2092             }
2093             if (getClass() != obj.getClass()) {
2094                 return false;
2095             }
2096             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2097             if (identifier == null) {
2098                 if (other.identifier != null) {
2099                     return false;
2100                 }
2101             } else if (!identifier.equals(other.identifier)) {
2102                 return false;
2103             }
2104             if (vpnInterface == null) {
2105                 if (other.vpnInterface != null) {
2106                     return false;
2107                 }
2108             } else if (!vpnInterface.equals(other.vpnInterface)) {
2109                 return false;
2110             }
2111             return true;
2112         }
2113     }
2114
2115     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2116         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2117         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2118         if (vpnInstanceOpData == null) {
2119             return;
2120         }
2121         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2122         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2123             return;
2124         }
2125         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2126         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2127             InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2128                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2129             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = VpnUtil.read(dataBroker,
2130                     LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2131             if (!vpnInterfaceOptional.isPresent()) {
2132                 return;
2133             }
2134             List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2135                     vpnInterface.getInterfaceName());
2136             if (configVpnAdjacencies == null) {
2137                 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2138                 return;
2139             }
2140             List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2141                     .getAugmentation(AdjacenciesOp.class).getAdjacency();
2142             // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2143             // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2144             // in operational DS. These unprocessed adjacencies will be handled below.
2145             // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2146             // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2147             configVpnAdjacencies.stream()
2148                     .filter(adjacency -> operationVpnAdjacencies.stream()
2149                             .noneMatch(operationalAdjacency ->
2150                                     operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2151                     .forEach(adjacency -> {
2152                         LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2153                         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2154                             () -> {
2155                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
2156                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
2157                                 if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2158                                     addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd, adjacency,
2159                                             vpnInterfaceOptional.get().getDpnId(), writeConfigTxn, writeOperTxn);
2160                                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
2161                                     try {
2162                                         operFuture.get();
2163                                     } catch (ExecutionException | InterruptedException e) {
2164                                         LOG.error("Exception encountered while submitting operational"
2165                                                 + " future for vpnInterface {}", vpnInterface, e);
2166                                     }
2167                                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2168                                     futures.add(writeConfigTxn.submit());
2169                                     return futures;
2170                                 }  else {
2171                                     return Collections.emptyList();
2172                                 }
2173                             });
2174                     });
2175         }));
2176     }
2177
2178     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2179         private final String interfaceName;
2180         private final boolean add;
2181         private final String txnDestination;
2182
2183         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2184             this.interfaceName = interfaceName;
2185             this.add = add;
2186             this.txnDestination = transactionDest;
2187         }
2188
2189         @Override
2190         public void onSuccess(Void voidObj) {
2191             if (add) {
2192                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2193                         interfaceName, txnDestination);
2194             } else {
2195                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2196             }
2197         }
2198
2199         @Override
2200         public void onFailure(Throwable throwable) {
2201             if (add) {
2202                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2203                         interfaceName, txnDestination, throwable);
2204             } else {
2205                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2206                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
2207             }
2208         }
2209     }
2210 }