vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.concurrent.ConcurrentHashMap;
  25 import java.util.concurrent.ConcurrentLinkedQueue;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  42 import org.opendaylight.genius.mdsalutil.NWUtil;
  43 import org.opendaylight.genius.mdsalutil.NwConstants;
  44 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.infrautils.caches.CacheProvider;
  47 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  51 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  52 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  53 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  54 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  55 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  56 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  57 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  58 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  59 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 109 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 110 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 111 import org.slf4j.Logger;
 112 import org.slf4j.LoggerFactory;
 113
 114 @Singleton
 115 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 116
 117     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 118     private static final short DJC_MAX_RETRIES = 3;
 119
 120     private final DataBroker dataBroker;
 121     private final ManagedNewTransactionRunner txRunner;
 122     private final IBgpManager bgpManager;
 123     private final IFibManager fibManager;
 124     private final IMdsalApiManager mdsalManager;
 125     private final IdManagerService idManager;
 126     private final OdlInterfaceRpcService ifaceMgrRpcService;
 127     private final VpnFootprintService vpnFootprintService;
 128     private final IInterfaceManager interfaceManager;
 129     private final IVpnManager vpnManager;
 130     private final ArpResponderHandler arpResponderHandler;
 131     private final JobCoordinator jobCoordinator;
 132
 133     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 134
 135     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 136             new ConcurrentHashMap<>();
 137
 138     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 139
 140     @Inject
 141     public VpnInterfaceManager(final DataBroker dataBroker,
 142                                final IBgpManager bgpManager,
 143                                final IdManagerService idManager,
 144                                final IMdsalApiManager mdsalManager,
 145                                final IFibManager fibManager,
 146                                final OdlInterfaceRpcService ifaceMgrRpcService,
 147                                final VpnFootprintService vpnFootprintService,
 148                                final IInterfaceManager interfaceManager,
 149                                final IVpnManager vpnManager,
 150                                final ArpResponderHandler arpResponderHandler,
 151                                final JobCoordinator jobCoordinator,
 152                                final CacheProvider cacheProvider) {
 153         super(VpnInterface.class, VpnInterfaceManager.class);
 154
 155         this.dataBroker = dataBroker;
 156         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 157         this.bgpManager = bgpManager;
 158         this.idManager = idManager;
 159         this.mdsalManager = mdsalManager;
 160         this.fibManager = fibManager;
 161         this.ifaceMgrRpcService = ifaceMgrRpcService;
 162         this.vpnFootprintService = vpnFootprintService;
 163         this.interfaceManager = interfaceManager;
 164         this.vpnManager = vpnManager;
 165         this.arpResponderHandler = arpResponderHandler;
 166         this.jobCoordinator = jobCoordinator;
 167
 168         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 169                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 170                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 171     }
 172
 173     public Runnable isNotifyTaskQueued(String intfName) {
 174         return vpnIntfMap.remove(intfName);
 175     }
 176
 177     @PostConstruct
 178     public void start() {
 179         LOG.info("{} start", getClass().getSimpleName());
 180         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 181     }
 182
 183     @Override
 184     @PreDestroy
 185     public void close() {
 186         super.close();
 187         vpnInstanceOpDataEntryCache.close();
 188     }
 189
 190     @Override
 191     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 192         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 193     }
 194
 195     @Override
 196     protected VpnInterfaceManager getDataTreeChangeListener() {
 197         return VpnInterfaceManager.this;
 198     }
 199
 200     @Override
 201     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 202         LOG.info("add: intfName {} onto vpnName {}",
 203                  vpnInterface.getName(),
 204                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 205         addVpnInterface(identifier, vpnInterface, null, null);
 206     }
 207
 208     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 209                           final VpnInterface vpnInterface, String vpnName) {
 210         synchronized (vpnName.intern()) {
 211             if (isVpnInstanceReady(vpnName)) {
 212                 return true;
 213             }
 214             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 215             return false;
 216         }
 217     }
 218
 219     // TODO Clean up the exception handling
 220     @SuppressWarnings("checkstyle:IllegalCatch")
 221     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 222                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 223         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 224             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 225             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 226         }
 227     }
 228
 229     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 230                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 231         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 232         final String interfaceName = key.getName();
 233
 234         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 235             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 236                   vpnName, vpnInterface.getName());
 237             return;
 238         }
 239         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 240              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 241         List<Adjacency> copyOldAdjs = null;
 242         if (oldAdjs != null) {
 243             copyOldAdjs = new ArrayList<>();
 244             copyOldAdjs.addAll(oldAdjs);
 245         }
 246         List<Adjacency> copyNewAdjs = null;
 247         if (newAdjs != null) {
 248             copyNewAdjs = new ArrayList<>();
 249             copyNewAdjs.addAll(newAdjs);
 250         }
 251         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 252     }
 253
 254     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 255                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 256                     final List<Adjacency> newAdjs,
 257                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 258         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 259         final String interfaceName = key.getName();
 260         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 261         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 262             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 263             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 264             if (interfaceState != null) {
 265                 try {
 266                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 267                     final int ifIndex = interfaceState.getIfIndex();
 268                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 269                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 270                         WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 271                         WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 272                         LOG.info("addVpnInterface: VPN Interface add event - intfName {} vpnName {} on dpn {}" ,
 273                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 274                         processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, writeConfigTxn,
 275                                      writeOperTxn, writeInvTxn, interfaceState, vpnName);
 276                         if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 277                             LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 278                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 279                             if (newAdjs != null) {
 280                                 for (Adjacency adj : newAdjs) {
 281                                     if (oldAdjs.contains(adj)) {
 282                                         oldAdjs.remove(adj);
 283                                     } else {
 284                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 285                                                                   dataBroker, adj)) {
 286                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd,
 287                                                   adj, dpnId, writeOperTxn, writeConfigTxn);
 288                                         }
 289                                     }
 290                                 }
 291                             }
 292                             for (Adjacency adj : oldAdjs) {
 293                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 294                                                           dataBroker, adj)) {
 295                                     delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 296                                         writeOperTxn, writeConfigTxn);
 297                                 }
 298                             }
 299                         }
 300                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
 301                         try {
 302                             operFuture.get();
 303                         } catch (ExecutionException e) {
 304                             LOG.error("addVpnInterface: Exception encountered while submitting operational future for"
 305                                     + " addVpnInterface {} on vpn {}", vpnInterface.getName(), vpnName, e);
 306                             return null;
 307                         }
 308                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 309                         ListenableFuture<Void> configFuture = writeConfigTxn.submit();
 310                         futures.add(configFuture);
 311                         Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"));
 312                         futures.add(writeInvTxn.submit());
 313                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 314                                 + " processed successfully", interfaceName, vpnName, dpnId);
 315                         return futures;
 316                     });
 317                 } catch (NumberFormatException | IllegalStateException e) {
 318                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 319                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 320                                     vpnName, e);
 321                     return;
 322                 }
 323             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 324                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 325                     () -> {
 326                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 327                         createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 328                                        writeConfigTxn, vpnName);
 329                         LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 330                                  vpnName, vpnInterface.getDpnId());
 331                         ListenableFuture<Void> futures = writeConfigTxn.submit();
 332                         String errorText = "addVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
 333                              + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
 334                         ListenableFutures.addErrorLogging(futures, LOG, errorText);
 335                         return Collections.singletonList(futures);
 336                     });
 337             } else {
 338                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 339                         + " is not available", interfaceName, vpnName);
 340             }
 341         } else {
 342             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 343                       + " as vpn is pending delete", interfaceName, vpnName,
 344                     vpnInterface.getDpnId());
 345         }
 346     }
 347
 348     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 349     // see comments below.
 350     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 351     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 352             final int lportTag, boolean isInterfaceUp,
 353             WriteTransaction writeConfigTxn,
 354             WriteTransaction writeOperTxn,
 355             WriteTransaction writeInvTxn,
 356             Interface interfaceState,
 357             final String vpnName) {
 358         final String interfaceName = vpnInterface.getName();
 359         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 360                                                          interfaceName, vpnName);
 361         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 362         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 363         if (!isInterfaceUp) {
 364             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 365                      interfaceName, dpId, vpnName);
 366             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 367             if (vpnId == VpnConstants.INVALID_ID) {
 368                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 369                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 370                         dpId);
 371                 return;
 372             }
 373
 374             boolean waitForVpnInterfaceOpRemoval = false;
 375             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 376                 String opVpnName = opVpnInterface.getVpnInstanceName();
 377                 String primaryInterfaceIp = null;
 378                 if (opVpnName.equals(vpnName)) {
 379                     // Please check if the primary VRF Entry does not exist for VPNInterface
 380                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 381                     // back to back
 382                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 383                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 384                     if (adjs == null) {
 385                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 386                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 387                                 vpnName);
 388                         return;
 389                     }
 390                     for (Adjacency adj : adjs) {
 391                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 392                             primaryInterfaceIp = adj.getIpAddress();
 393                             break;
 394                         }
 395                     }
 396                     if (primaryInterfaceIp == null) {
 397                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 398                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 399                                 dpId, vpnName);
 400                         return;
 401                     }
 402                     // Get the rd of the vpn instance
 403                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 404                     if (vrf != null) {
 405                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 406                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 407                         return;
 408                     }
 409                     waitForVpnInterfaceOpRemoval = true;
 410                 } else {
 411                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 412                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 413                 }
 414             }
 415             if (!waitForVpnInterfaceOpRemoval) {
 416                 // Add the VPNInterface and quit
 417                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 418                         null/*ipAddressSourceValuePair*/,
 419                         true /* add */);
 420                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 421                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 422                 if (!isBgpVpnInternetVpn) {
 423                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 424                             jobCoordinator);
 425                 }
 426                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 427                         dpId, vpnName);
 428                 if (interfaceManager.isExternalInterface(interfaceName)) {
 429                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 430                         NwConstants.ADD_FLOW);
 431                 }
 432                 return;
 433             }
 434
 435             // FIB didn't get a chance yet to clean up this VPNInterface
 436             // Let us give it a chance here !
 437             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 438                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 439             try {
 440                 Runnable notifyTask = new VpnNotifyTask();
 441                 synchronized (notifyTask) {
 442                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 443                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 444                     // published here while holding the lock on it so this path will hit the wait before notify can be
 445                     // invoked.
 446                     vpnIntfMap.put(interfaceName, notifyTask);
 447                     try {
 448                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 449                     } catch (InterruptedException e) {
 450                         // Ignored
 451                     }
 452                 }
 453             } finally {
 454                 vpnIntfMap.remove(interfaceName);
 455             }
 456
 457             if (opVpnInterface != null) {
 458                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 459                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 460                         dpId, vpnName);
 461                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
 462                 return;
 463             }
 464             // VPNInterface got removed, proceed with Add
 465             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 466                     interfaceName, dpId, vpnName);
 467             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 468                     null/*ipAddressSourceValuePair*/,
 469                     true /* add */);
 470             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 471                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 472             if (!isBgpVpnInternetVpn) {
 473                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 474             }
 475             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 476                     + " FIB to clean up", interfaceName, dpId, vpnName);
 477             if (interfaceManager.isExternalInterface(interfaceName)) {
 478                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 479                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 480             }
 481         } else {
 482             // Interface is retained in the DPN, but its Link Up.
 483             // Advertise prefixes again for this interface to BGP
 484             InstanceIdentifier<VpnInterface> identifier =
 485                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 486             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 487                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 488             advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 489             // Perform similar operation as interface add event for extraroutes.
 490             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 491             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 492             if (!optAdjacencies.isPresent()) {
 493                 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 494                 return;
 495             }
 496             List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 497             for (Adjacency adjacency : adjacencies) {
 498                 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 499                     continue;
 500                 }
 501                 // if BGPVPN Internet, filter only IPv6 Adjacencies
 502                 if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 503                                 dataBroker, adjacency)) {
 504                     continue;
 505                 }
 506                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 507                           dpId, writeOperTxn, writeConfigTxn);
 508             }
 509         }
 510     }
 511
 512     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 513             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 514         Uuid extNetworkId;
 515         try {
 516             // vpn instance of ext-net interface is the network-id
 517             extNetworkId = new Uuid(vpnName);
 518         } catch (IllegalArgumentException e) {
 519             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 520                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 521             return;
 522         }
 523
 524         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 525         if (routerIds == null || routerIds.isEmpty()) {
 526             LOG.info("processExternalVpnInterface: No router is associated with {}."
 527                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 528                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 529             return;
 530         }
 531
 532         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 533                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 534         for (Uuid routerId : routerIds) {
 535             String routerName = routerId.getValue();
 536             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 537             if (Objects.equals(primarySwitch, dpId)) {
 538                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 539                 if (router != null) {
 540                     if (addOrRemove == NwConstants.ADD_FLOW) {
 541                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 542                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 543                                 dpId, interfaceName, lportTag, writeInvTxn);
 544                     } else {
 545                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 546                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 547                                 dpId, interfaceName, lportTag);
 548                     }
 549                 } else {
 550                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 551                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 552                             interfaceName, dpId, vpnName);
 553                 }
 554             }
 555         }
 556     }
 557
 558     // TODO Clean up the exception handling
 559     @SuppressWarnings("checkstyle:IllegalCatch")
 560     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 561                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 562                                                   String vpnName, String interfaceName) {
 563         if (rd == null) {
 564             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 565                   interfaceName, dpnId, vpnName);
 566             return;
 567         } else {
 568             if (rd.equals(vpnName)) {
 569                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 570                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 571                 return;
 572             }
 573         }
 574         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 575                 interfaceName, dpnId, vpnName, rd);
 576
 577         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 578         if (nextHopIp == null) {
 579             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 580                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 581                     rd, vpnName);
 582             return;
 583         }
 584
 585         //Read NextHops
 586         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 587         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 588         if (adjacencies.isPresent()) {
 589             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 590
 591             if (!nextHops.isEmpty()) {
 592                 LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 593                         + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 594                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 595                 long l3vni = vpnInstanceOpData.getL3vni();
 596                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 597                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 598                 for (Adjacency nextHop : nextHops) {
 599                     if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 600                         continue;
 601                     }
 602                     String gatewayMac = null;
 603                     long label = 0;
 604                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 605                         final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 606                             vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 607                         gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName).get();
 608                     } else {
 609                         label = nextHop.getLabel();
 610                     }
 611                     try {
 612                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 613                                 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 614                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 615                                 encapType, (int)label, l3vni, 0 /*l2vni*/,
 616                                 gatewayMac);
 617                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 618                                 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 619                                 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 620                     } catch (Exception e) {
 621                         LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}"
 622                                 + " for interface {} on dpn {}", nextHop.getIpAddress(),
 623                                   vpnName, rd, interfaceName, dpnId, e);
 624                     }
 625                 }
 626             }
 627         }
 628     }
 629
 630     // TODO Clean up the exception handling
 631     @SuppressWarnings("checkstyle:IllegalCatch")
 632     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 633                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 634         //Read NextHops
 635         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 636         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 637
 638         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 639         if (rd == null) {
 640             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 641                 interfaceName, vpnName);
 642             return;
 643         } else {
 644             if (rd.equals(vpnName)) {
 645                 LOG.info(
 646                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 647                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 648                 return;
 649             }
 650         }
 651         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 652                vpnName, rd);
 653         if (adjacencies.isPresent()) {
 654             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 655
 656             if (!nextHops.isEmpty()) {
 657                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 658                         nextHops, interfaceName, vpnName, rd);
 659                 for (Adjacency nextHop : nextHops) {
 660                     try {
 661                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 662                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 663                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 664                                     interfaceName, vpnName);
 665                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 666                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 667                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 668                                     interfaceName, vpnName);
 669                         } else {
 670                             // Perform similar operation as interface delete event for extraroutes.
 671                             String allocatedRd = nextHop.getVrfId();
 672                             for (String nh : nextHop.getNextHopIpList()) {
 673                                 deleteExtraRouteFromCurrentAndImportingVpns(
 674                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 675                             }
 676                         }
 677                     } catch (Exception e) {
 678                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 679                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 680                     }
 681                 }
 682             }
 683         }
 684     }
 685
 686     @SuppressWarnings("checkstyle:IllegalCatch")
 687     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 688                                                   String primaryRd, String interfaceName, final long vpnId,
 689                                                   WriteTransaction writeConfigTxn,
 690                                                   WriteTransaction writeOperTxn,
 691                                                   final WriteTransaction writeInvTxn,
 692                                                   Interface interfaceState) {
 693         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 694         // Read NextHops
 695         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 696         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 697         if (!adjacencies.isPresent()) {
 698             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 699                     null/*gwMac*/, writeOperTxn);
 700             return;
 701         }
 702
 703         // Get the rd of the vpn instance
 704         String nextHopIp = null;
 705         try {
 706             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 707         } catch (Exception e) {
 708             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 709                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 710         }
 711         List<String> nhList = new ArrayList<>();
 712         if (nextHopIp != null) {
 713             nhList.add(nextHopIp);
 714             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 715                     interfaceName, dpnId, vpnName, nhList);
 716         }
 717         Optional<String> gwMac = Optional.absent();
 718         String vpnInterfaceSubnetGwMacAddress = null;
 719         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 720         Long l3vni = vpnInstanceOpData.getL3vni();
 721         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 722         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 723         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 724         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 725         List<Adjacency> value = new ArrayList<>();
 726         for (Adjacency nextHop : nextHops) {
 727             String rd = primaryRd;
 728             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 729             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 730                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 731                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 732                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 733                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 734                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 735                 continue;
 736             }
 737             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 738                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 739                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 740                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 741                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 742                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 743                 writeOperTxn.merge(
 744                     LogicalDatastoreType.OPERATIONAL,
 745                     VpnUtil.getPrefixToInterfaceIdentifier(
 746                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 747                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 748                             prefixCue), true);
 749                 final Uuid subnetId = nextHop.getSubnetId();
 750
 751                 String gatewayIp = nextHop.getSubnetGatewayIp();
 752                 if (gatewayIp == null) {
 753                     Optional<String> gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 754                     if (gatewayIpOptional.isPresent()) {
 755                         gatewayIp = gatewayIpOptional.get();
 756                     }
 757                 }
 758
 759                 if (gatewayIp != null) {
 760                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 761                     if (gwMac.isPresent()) {
 762                         // A valid mac-address is available for this subnet-gateway-ip
 763                         // Use this for programming ARP_RESPONDER table here.  And save this
 764                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 765                         // to populate L3_GW_MAC_TABLE there.
 766                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 767                                 gatewayIp, gwMac.get());
 768                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 769                     } else {
 770                         // A valid mac-address is not available for this subnet-gateway-ip
 771                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 772                         // Save this connected-mac-address as gateway-mac-address for the
 773                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 774                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 775                         if (gwMac.isPresent()) {
 776                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 777                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 778                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 779                                     gatewayIp, gwMac.get());
 780                         } else {
 781                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 782                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 783                                 + "gwIp {}",
 784                                 subnetId, interfaceName, vpnName, gatewayIp);
 785                         }
 786                     }
 787                 } else {
 788                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 789                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 790                         subnetId, interfaceName, vpnName);
 791                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 792                 }
 793                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 794                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 795             } else {
 796                 //Extra route adjacency
 797                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 798                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 799                 synchronized (vpnPrefixKey.intern()) {
 800                     java.util.Optional<String> rdToAllocate = VpnUtil
 801                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 802                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 803                     if (rdToAllocate.isPresent()) {
 804                         rd = rdToAllocate.get();
 805                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 806                             rd, prefix);
 807                     } else {
 808                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 809                         continue;
 810                     }
 811                 }
 812                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 813                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 814                         interfaceName, dpnId);
 815             }
 816             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 817             // can be different from the gateway-mac-address within the VRFEntry as the
 818             // gateway-mac-address is a superset.
 819             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 820                     : RouteOrigin.STATIC;
 821             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 822                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 823                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 824             Adjacency operationalAdjacency = null;
 825             try {
 826                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 827             } catch (NullPointerException e) {
 828                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 829                     input, e.getMessage());
 830                 return;
 831             }
 832             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 833                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 834                         vpnName, l3vni, origin,
 835                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 836             }
 837             value.add(operationalAdjacency);
 838         }
 839
 840         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 841         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 842                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 843
 844         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 845                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 846                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 847
 848         for (Adjacency nextHop : aug.getAdjacency()) {
 849             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 850             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 851                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 852                         : RouteOrigin.STATIC;
 853                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 854                 registeredPopulator.populateFib(input, writeConfigTxn);
 855             }
 856         }
 857     }
 858
 859     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 860                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 861         VpnInterfaceOpDataEntry opInterface =
 862               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 863         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 864             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 865         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 866                 WriteTransaction.CREATE_MISSING_PARENTS);
 867         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 868                 interfaceName, dpnId, vpnName);
 869     }
 870
 871     // TODO Clean up the exception handling
 872     @SuppressWarnings("checkstyle:IllegalCatch")
 873     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 874                                            StateTunnelList stateTunnelList,
 875                                            WriteTransaction writeConfigTxn,
 876                                            WriteTransaction writeOperTxn) {
 877
 878         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 879         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 880         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 881         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 882         if (adjList.isEmpty()) {
 883             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 884                     vpnInterface, srcDpnId);
 885             return;
 886         }
 887         String prefix = null;
 888         long label = 0;
 889         List<Adjacency> value = new ArrayList<>();
 890         boolean isNextHopAddReqd = false;
 891         String vpnName = vpnInterface.getVpnInstanceName();
 892         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 893         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 894         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 895                 vpnInterface.getName(), vpnInterface.getDpnId(),
 896                 vpnInterface.getVpnInstanceName(), adjList);
 897         for (Adjacency adj : adjList) {
 898             String rd = adj.getVrfId();
 899             rd = rd != null ? rd : vpnName;
 900             prefix = adj.getIpAddress();
 901             label = adj.getLabel();
 902             List<String> nhList = Collections.singletonList(srcTepIp);
 903             List<String> nextHopList = adj.getNextHopIpList();
 904             // If TEP is added , update the nexthop of primary adjacency.
 905             // Secondary adj nexthop is already pointing to primary adj IP address.
 906             if (nextHopList != null && !nextHopList.isEmpty() && nextHopList.get(0).equalsIgnoreCase(srcTepIp)) {
 907                 /* everything right already */
 908             } else {
 909                 isNextHopAddReqd = true;
 910             }
 911
 912             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 913                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 914             } else {
 915                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 916                 if (!vrfEntryOptional.isPresent()) {
 917                     continue;
 918                 }
 919                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 920                 if (!nhList.contains(srcTepIp)) {
 921                     nhList.add(srcTepIp);
 922                     isNextHopAddReqd = true;
 923                 }
 924                 value.add(adj);
 925             }
 926
 927             if (isNextHopAddReqd) {
 928                 updateLabelMapper(label, nhList);
 929                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 930                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 931                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 932                 // Update the VRF entry with nextHop
 933                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 934                         label, true, writeConfigTxn);
 935
 936                 //Get the list of VPN's importing this route(prefix) .
 937                 // Then update the VRF entry with nhList
 938                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 939                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 940                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 941                     String vpnRd = vpn.getVrfId();
 942                     if (vpnRd != null) {
 943                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 944                                 srcTepIp, label, true, writeConfigTxn);
 945                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 946                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 947                             vpnInterface.getName(), srcDpnId, vpnName,
 948                             vpn.getVpnInstanceName(), vpnRd);
 949                     }
 950                 }
 951                 // Advertise the prefix to BGP only for external vpn
 952                 // since there is a nexthop change.
 953                 try {
 954                     if (!rd.equalsIgnoreCase(vpnName)) {
 955                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 956                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 957                                 null /*gatewayMacAddress*/);
 958                     }
 959                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 960                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 961                             srcDpnId, vpnName);
 962                 } catch (Exception ex) {
 963                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 964                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 965                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 966                 }
 967             }
 968         }
 969         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 970         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 971                 .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 972                 .addAugmentation(AdjacenciesOp.class, aug).build();
 973         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 974                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 975         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 976                 WriteTransaction.CREATE_MISSING_PARENTS);
 977         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 978                 vpnInterface.getName(), srcDpnId, vpnName);
 979
 980     }
 981
 982     // TODO Clean up the exception handling
 983     @SuppressWarnings("checkstyle:IllegalCatch")
 984     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
 985                                               StateTunnelList stateTunnelList,
 986                                               WriteTransaction writeConfigTxn,
 987                                               WriteTransaction writeOperTxn) {
 988
 989         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 990         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 991         String prefix = null;
 992         long label = 0;
 993         boolean isNextHopRemoveReqd = false;
 994         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 995         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 996         String vpnName = vpnInterface.getVpnInstanceName();
 997         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 998         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 999         if (adjList != null) {
1000             List<Adjacency> value = new ArrayList<>();
1001             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1002                     vpnInterface.getName(), vpnInterface.getDpnId(),
1003                     vpnInterface.getVpnInstanceName(), adjList);
1004             for (Adjacency adj : adjList) {
1005                 List<String> nhList = new ArrayList<>();
1006                 String rd = adj.getVrfId();
1007                 rd = rd != null ? rd : vpnName;
1008                 prefix = adj.getIpAddress();
1009                 List<String> nextHopList = adj.getNextHopIpList();
1010                 label = adj.getLabel();
1011                 if (nextHopList != null && !nextHopList.isEmpty()) {
1012                     isNextHopRemoveReqd = true;
1013                 }
1014                 // If TEP is deleted , remove the nexthop from primary adjacency.
1015                 // Secondary adj nexthop will continue to point to primary adj IP address.
1016                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1017                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1018                 } else {
1019                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1020                     if (!vrfEntryOptional.isPresent()) {
1021                         continue;
1022                     }
1023                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1024                     if (nhList.contains(srcTepIp)) {
1025                         nhList.remove(srcTepIp);
1026                         isNextHopRemoveReqd = true;
1027                     }
1028                     value.add(adj);
1029                 }
1030
1031                 if (isNextHopRemoveReqd) {
1032                     updateLabelMapper(label, nhList);
1033                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1034                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1035                             prefix, nhList, vpnName,
1036                             vpnId, rd, vpnInterface.getName());
1037                     // Update the VRF entry with removed nextHop
1038                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1039                             label, false, writeConfigTxn);
1040
1041                     //Get the list of VPN's importing this route(prefix) .
1042                     // Then update the VRF entry with nhList
1043                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1044                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1045                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1046                         String vpnRd = vpn.getVrfId();
1047                         if (vpnRd != null) {
1048                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1049                                     srcTepIp, label, false, writeConfigTxn);
1050                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1051                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1052                                     nhList, label, vpnInterface.getName(), srcDpnId,
1053                                     vpnName,
1054                                     vpn.getVpnInstanceName(), vpnRd);
1055                         }
1056                     }
1057
1058                     // Withdraw prefix from BGP only for external vpn.
1059                     try {
1060                         if (!rd.equalsIgnoreCase(vpnName)) {
1061                             bgpManager.withdrawPrefix(rd, prefix);
1062                         }
1063                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1064                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1065                                 vpnInterface.getName(), srcDpnId,
1066                                 vpnName);
1067                     } catch (Exception ex) {
1068                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1069                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1070                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1071                     }
1072                 }
1073             }
1074             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1075             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1076                     .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1077                     .addAugmentation(AdjacenciesOp.class, aug).build();
1078             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1079                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1080             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1081                     WriteTransaction.CREATE_MISSING_PARENTS);
1082             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1083                          vpnInterface.getName(), srcDpnId, vpnName);
1084         }
1085     }
1086
1087     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1088         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1089
1090         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1091         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1092         if (vpnInstanceOpDataEntry == null) {
1093             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1094                     + " to check for vpns exporting the routes", vpnName);
1095             return vpnsToExportRoute;
1096         }
1097
1098         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1099             if (input.getVpnInstanceName() == null) {
1100                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1101                         input.getVrfId());
1102                 return false;
1103             }
1104             return !input.getVpnInstanceName().equals(vpnName);
1105         };
1106
1107         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1108             Iterable<String> commonRTs =
1109                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1110                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1111             return Iterators.size(commonRTs.iterator()) > 0;
1112         };
1113
1114         vpnsToExportRoute =
1115                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1116                         Collectors.toList());
1117         return vpnsToExportRoute;
1118     }
1119
1120     // TODO Clean up the exception handling
1121     @SuppressWarnings("checkstyle:IllegalCatch")
1122     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1123         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1124         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1125             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1126             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1127             if (vrfEntries != null) {
1128                 for (VrfEntry vrfEntry : vrfEntries) {
1129                     try {
1130                         if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1131                                 RouteOrigin.value(vrfEntry.getOrigin()))) {
1132                             LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1133                                     + " is not a controller managed non intervpn link route. Ignoring.",
1134                                     vpn.getVrfId(), vrfEntry.getDestPrefix());
1135                             continue;
1136                         }
1137                         String prefix = vrfEntry.getDestPrefix();
1138                         String gwMac = vrfEntry.getGatewayMacAddress();
1139                         vrfEntry.getRoutePaths().forEach(routePath -> {
1140                             String nh = routePath.getNexthopAddress();
1141                             int label = routePath.getLabel().intValue();
1142                             if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1143                                     vrfEntry.getOrigin()))) {
1144                                 LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1145                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1146                                         vpnName, vpnRd);
1147                                 fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1148                                         Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1149                                         0 /*l3vni*/, gwMac,  null /*parentVpnRd*/, RouteOrigin.SELF_IMPORTED,
1150                                         writeConfigTxn);
1151                             } else {
1152                                 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} prefix {}"
1153                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1154                                         vpnName, vpnRd);
1155                                 SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1156                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, writeConfigTxn);
1157                             }
1158                         });
1159                     } catch (RuntimeException e) {
1160                         LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1161                                 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1162                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1163                     }
1164                 }
1165                 writeConfigTxn.submit();
1166             } else {
1167                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1168                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1169             }
1170         }
1171     }
1172
1173     @SuppressWarnings("checkstyle:IllegalCatch")
1174     @Override
1175     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1176         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1177         final String interfaceName = key.getName();
1178         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1179             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1180             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1181         }
1182     }
1183
1184     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1185                                 final VpnInterface vpnInterface, final String vpnName,
1186                                 final String interfaceName) {
1187         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1188             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1189                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1190                 deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn, vpnName);
1191                 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1192                 ListenableFuture<Void> futures = writeConfigTxn.submit();
1193                 String errorText = "removeVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
1194                         + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
1195                 ListenableFutures.addErrorLogging(futures, LOG, errorText);
1196                 return Collections.singletonList(futures);
1197             }, DJC_MAX_RETRIES);
1198         } else {
1199             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1200             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1201         }
1202     }
1203
1204     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1205     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1206                                 final VpnInterface vpnInterface, final String vpnName,
1207                                 final String interfaceName, final Interface interfaceState) {
1208         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1209                 vpnName, vpnInterface.getDpnId());
1210         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1211         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1212             () -> {
1213                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1214                 ListenableFuture<Void> configFuture = txRunner
1215                         .callWithNewWriteOnlyTransactionAndSubmit(writeConfigTxn -> {
1216                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeOperTxn -> {
1217                                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1218                                     LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1219                                             interfaceName, vpnName);
1220                                     BigInteger dpId = BigInteger.ZERO;
1221                                     int ifIndex = 0;
1222                                     String gwMacAddress = null;
1223                                     InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1224                                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1225                                     final Optional<VpnInterfaceOpDataEntry> optVpnInterface =
1226                                             VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1227                                     if (interfaceState != null) {
1228                                         try {
1229                                             dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1230                                         } catch (NumberFormatException | IllegalStateException e) {
1231                                             LOG.error("remove: Unable to retrieve dpnId from interface operational"
1232                                                     + " data store for interface {} on dpn {} for vpn {} Fetching"
1233                                                             + " from vpn interface op data store. ", interfaceName,
1234                                                     vpnInterface.getDpnId(), vpnName, e);
1235                                             dpId = BigInteger.ZERO;
1236                                         }
1237                                         ifIndex = interfaceState.getIfIndex();
1238                                         gwMacAddress = interfaceState.getPhysAddress().getValue();
1239                                     } else {
1240                                         LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1241                                                 + " from vpn interface op.", interfaceName);
1242                                         if (optVpnInterface.isPresent()) {
1243                                             VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1244                                             dpId = vpnOpInterface.getDpnId();
1245                                             ifIndex = vpnOpInterface.getLportTag().intValue();
1246                                             gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1247                                         } else {
1248                                             LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1249                                                             + " as interfaceState and vpn interface op is not"
1250                                                     + " available", interfaceName, vpnName);
1251                                             return;
1252                                         }
1253                                     }
1254                                     processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1255                                             optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1256                                             writeConfigTxn, writeOperTxn, writeInvTxn);
1257                                     LOG.info(
1258                                             "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1259                                                     + "successfully",
1260                                             interfaceName, vpnInterface.getDpnId(), vpnName);
1261                                 }));
1262                             }));
1263                         });
1264                 futures.add(configFuture);
1265                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1266                 return futures;
1267             }, DJC_MAX_RETRIES);
1268     }
1269
1270     protected void processVpnInterfaceDown(BigInteger dpId,
1271                                            String interfaceName,
1272                                            int lportTag,
1273                                            String gwMac,
1274                                            VpnInterfaceOpDataEntry vpnOpInterface,
1275                                            boolean isInterfaceStateDown,
1276                                            WriteTransaction writeConfigTxn,
1277                                            WriteTransaction writeOperTxn,
1278                                            WriteTransaction writeInvTxn) {
1279         if (vpnOpInterface == null) {
1280             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1281                     + " as it is not available in operational data store", interfaceName, dpId);
1282             return;
1283         }
1284         final String vpnName = vpnOpInterface.getVpnInstanceName();
1285         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1286                                                     interfaceName, vpnName);
1287         if (!isInterfaceStateDown) {
1288             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1289             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1290                     null);
1291             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1292             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1293                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1294             if (interfaceManager.isExternalInterface(interfaceName)) {
1295                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1296                         NwConstants.DEL_FLOW);
1297             }
1298             if (!isBgpVpnInternetVpn) {
1299                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1300             }
1301             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1302                     + " successful", interfaceName, dpId, vpnName);
1303         } else {
1304             // Interface is retained in the DPN, but its Link Down.
1305             // Only withdraw the prefixes for this interface from BGP
1306             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1307         }
1308     }
1309
1310     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1311                                           final String vpnName, final long vpnId, String gwMac,
1312                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1313                                           final WriteTransaction writeInvTxn) {
1314         //Read NextHops
1315         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1316                 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1317         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1318         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1319
1320         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1321         LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1322                 interfaceName, dpnId, vpnName, primaryRd);
1323         if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1324             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1325             LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1326                     interfaceName, dpnId, vpnName, nextHops);
1327             for (Adjacency nextHop : nextHops) {
1328                 if (nextHop.isPhysNetworkFunc()) {
1329                     LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1330                             nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1331                     fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1332                             null/*writeCfgTxn*/);
1333                 } else {
1334                     String rd = nextHop.getVrfId();
1335                     List<String> nhList;
1336                     if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1337                         nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1338                     } else {
1339                         // This is a primary adjacency
1340                         nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1341                                 : Collections.emptyList();
1342                         removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1343                                 interfaceName, writeInvTxn);
1344                     }
1345                     if (!nhList.isEmpty()) {
1346                         if (rd.equals(vpnName)) {
1347                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1348                             //remove from FIB directly
1349                             nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1350                                     interfaceName, dpnId, writeConfigTxn));
1351                         } else {
1352                             removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, interfaceName,
1353                                     writeConfigTxn);
1354                         }
1355                     } else {
1356                         LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1357                                         + " interface {}", nextHop.getIpAddress(), rd,
1358                                 nextHop.getAdjacencyType().toString(), interfaceName);
1359                         bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1360                         fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1361                     }
1362                 }
1363                 String ip = nextHop.getIpAddress().split("/")[0];
1364                 LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1365                 if (vpnVipToPort != null) {
1366                     VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip);
1367                     LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1368                                     + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(),
1369                             ip, dpnId, vpnName);
1370                 }
1371             }
1372         } else {
1373             // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1374             LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1375                     + " Removing it.", interfaceName, vpnName, dpnId);
1376             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1377         }
1378     }
1379
1380     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1381                                                                    String interfaceName, BigInteger dpnId,
1382                                                                    WriteTransaction writeConfigTxn) {
1383         return (nh) -> {
1384             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1385                     writeConfigTxn);
1386             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1387                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1388                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1389         };
1390     }
1391
1392     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1393                                            BigInteger dpnId, String rd, String interfaceName,
1394                                            WriteTransaction writeConfigTxn) {
1395         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1396                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1397         nhList.forEach((nh) -> {
1398             //IRT: remove routes from other vpns importing it
1399             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1400                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1401             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1402                 String vpnRd = vpn.getVrfId();
1403                 if (vpnRd != null) {
1404                     fibManager.removeOrUpdateFibEntry(vpnRd,
1405                             nextHop.getIpAddress(), nh, writeConfigTxn);
1406                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1407                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1408                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1409                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1410                 }
1411             }
1412         });
1413     }
1414
1415     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1416                                                  int lportTag, String gwMac, String interfaceName,
1417                                                  WriteTransaction writeInvTxn) {
1418         final Uuid subnetId = nextHop.getSubnetId();
1419         if (nextHop.getSubnetGatewayMacAddress() == null) {
1420             // A valid mac-address was not available for this subnet-gateway-ip
1421             // So a connected-mac-address was used for this subnet and we need
1422             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1423             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1424                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1425         }
1426         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1427                 subnetId);
1428     }
1429
1430     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1431                                                   String interfaceName) {
1432         // This is either an extra-route (or) a learned IP via subnet-route
1433         List<String> nhList = null;
1434         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1435         if (nextHopIp == null || nextHopIp.isEmpty()) {
1436             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1437                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1438                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1439                     vpnName);
1440             nhList = Collections.emptyList();
1441         } else {
1442             nhList = Collections.singletonList(nextHopIp);
1443         }
1444         return nhList;
1445     }
1446
1447     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1448         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1449         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1450         // else use connected interface
1451         if (gwPort != null && gwPort.isSubnetIp()) {
1452             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1453                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1454             return Optional.of(gwPort.getMacAddress());
1455         }
1456         return Optional.absent();
1457     }
1458
1459     @Override
1460     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1461         final VpnInterface update) {
1462         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1463                 update.getDpnId(), original.getVpnInstanceNames(),
1464                 update.getVpnInstanceNames());
1465         final String vpnInterfaceName = update.getName();
1466         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1467         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1468         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1469             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1470         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1471         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1472             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1473
1474         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1475         //handles switching between <internal VPN - external VPN>
1476         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1477             LOG.info("update: handled VPNInterface {} on dpn {} update"
1478                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1479                      original.getName(), dpnId,
1480                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1481                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1482             return;
1483         }
1484         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1485             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1486             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1487             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1488             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1489             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1490                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1491                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1492                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1493                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1494                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1495                     LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1496                             update.getName(), newVpnName);
1497                     //handle both addition and removal of adjacencies
1498                     //currently, new adjacency may be an extra route
1499                     boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1500                     if (!oldAdjs.equals(newAdjs)) {
1501                         for (Adjacency adj : copyNewAdjs) {
1502                             if (copyOldAdjs.contains(adj)) {
1503                                 copyOldAdjs.remove(adj);
1504                             } else {
1505                                 // add new adjacency - right now only extra route will hit this path
1506                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1507                                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1508                                              dpnId, writeOperTxn, writeConfigTxn);
1509                                 }
1510                                 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to vpn "
1511                                                 + "interface {} on vpn {} dpnId {}",
1512                                         adj.getIpAddress(), adj.getNextHopIpList(),
1513                                         adj.getLabel(), adj.getSubnetId(), update.getName(),
1514                                         newVpnName, dpnId);
1515                             }
1516                         }
1517                         for (Adjacency adj : copyOldAdjs) {
1518                             if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1519                                 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1520                                        writeOperTxn, writeConfigTxn);
1521                             }
1522                             LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1523                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1524                                 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1525                         }
1526                     }
1527                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
1528                     try {
1529                         operFuture.get();
1530                     } catch (ExecutionException e) {
1531                         LOG.error("Exception encountered while submitting operational future for update"
1532                                 + " VpnInterface {} on vpn {}", vpnInterfaceName, newVpnName, e);
1533                         return null;
1534                     }
1535                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1536                     futures.add(writeConfigTxn.submit());
1537                     LOG.info("update: vpn interface updated for interface {} oldVpn(s) {} newVpn {}"
1538                         + "processed successfully", update.getName(),
1539                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), newVpnName);
1540                     return futures;
1541                 });
1542             } else {
1543                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1544                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1545             }
1546         }
1547     }
1548
1549     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1550                                                  VpnInterface original, VpnInterface update) {
1551         boolean isSwap = Boolean.FALSE;
1552         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1553         final String interfaceName = key.getName();
1554         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1555             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1556         List<String> oldVpnListCopy = new ArrayList<>();
1557         oldVpnListCopy.addAll(oldVpnList);
1558         List<String> newVpnList = update.getVpnInstanceNames().stream()
1559             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1560         oldVpnList.removeAll(newVpnList);
1561         newVpnList.removeAll(oldVpnListCopy);
1562         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1563             for (String oldVpnName: oldVpnList) {
1564                 isSwap = Boolean.TRUE;
1565                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1566                         + " running config-driven swap removal", interfaceName, oldVpnName);
1567                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1568                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1569                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1570             }
1571             //Wait for previous interface bindings to be removed
1572             try {
1573                 Thread.sleep(2000);
1574             } catch (InterruptedException e) {
1575                 //Ignore
1576             }
1577             for (String newVpnName: newVpnList) {
1578                 String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1579                 isSwap = Boolean.TRUE;
1580                 if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1581                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1582                             + "running config-driven swap addition", interfaceName, newVpnName);
1583                     final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1584                     final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1585                             ? origAdjs.getAdjacency() : new ArrayList<>();
1586                     final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1587                     final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1588                             ? updateAdjs.getAdjacency() : new ArrayList<>();
1589
1590                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1591                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1592                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1593                             interfaceName, oldVpnListCopy, newVpnName);
1594                 }
1595             }
1596         }
1597         return isSwap;
1598     }
1599
1600     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1601         Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1602         synchronized (label.toString().intern()) {
1603             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1604                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1605             Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1606             if (opResult.isPresent()) {
1607                 LabelRouteInfo labelRouteInfo =
1608                     new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1609                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1610             }
1611         }
1612         LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList);
1613     }
1614
1615     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1616         SubnetRoute route, WriteTransaction writeConfigTxn) {
1617
1618         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1619         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, null /* parentVpnRd */)
1620                 .addAugmentation(SubnetRoute.class, route).build();
1621         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1622         InstanceIdentifierBuilder<VrfTables> idBuilder =
1623             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1624         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1625         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1626         if (writeConfigTxn != null) {
1627             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1628         } else {
1629             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1630         }
1631         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1632                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1633     }
1634
1635     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1636                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1637                                            WriteTransaction writeConfigTxn) {
1638
1639         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1640                                                 LogicalDatastoreType.OPERATIONAL, identifier);
1641
1642         if (optVpnInterface.isPresent()) {
1643             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1644             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1645             String vpnName = currVpnIntf.getVpnInstanceName();
1646             VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1647             InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1648             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker,
1649                                             LogicalDatastoreType.OPERATIONAL, adjPath);
1650             boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1651             VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1652             long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1653             VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1654             List<Adjacency> adjacencies;
1655             if (optAdjacencies.isPresent()) {
1656                 adjacencies = optAdjacencies.get().getAdjacency();
1657             } else {
1658                 // This code will be hit in case of first PNF adjacency
1659                 adjacencies = new ArrayList<>();
1660             }
1661             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1662             L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1663                     .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1664             Adjacency operationalAdjacency = null;
1665             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) {
1666                 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1667                         : RouteOrigin.STATIC;
1668                 String nh = adj.getNextHopIpList().get(0);
1669                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1670                 synchronized (vpnPrefixKey.intern()) {
1671                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1672                             dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1673                     if (rdToAllocate.isPresent()) {
1674                         input.setRd(rdToAllocate.get());
1675                         operationalAdjacency = populator.createOperationalAdjacency(input);
1676                         int label = operationalAdjacency.getLabel().intValue();
1677                         vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1678                                 currVpnIntf.getVpnInstanceName(), l3vni, origin,
1679                                 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1680                         LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1681                                 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1682                                 vpnName, label, currVpnIntf.getName(), dpnId);
1683                     } else {
1684                         LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName,
1685                                 prefix);
1686                         return;
1687                     }
1688                     // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1689                     // Keeping the MPLS check for now.
1690                     if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1691                         final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1692                         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1693                                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1694                         vpnsToImportRoute.forEach(vpn -> {
1695                             if (vpn.getVrfId() != null) {
1696                                 VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1697                                         dataBroker, vpn.getVpnId(), vpnId, prefix,
1698                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1699                                         .ifPresent(
1700                                             rds -> vpnManager.addExtraRoute(
1701                                                     VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1702                                                     adj.getIpAddress(), nh, rds,
1703                                                     currVpnIntf.getVpnInstanceName(),
1704                                                     l3vni, RouteOrigin.SELF_IMPORTED,
1705                                                     currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn));
1706                             }
1707                         });
1708                     }
1709                 }
1710             } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1711                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1712                         currVpnIntf.getName(), vpnName);
1713
1714                 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1715
1716                 writeOperTxn.merge(
1717                         LogicalDatastoreType.OPERATIONAL,
1718                         VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1719                                 adj.getSubnetId().getValue()), prefix),
1720                         VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1721                                 adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1722
1723                 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1724                         adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, 0 /*l3vni*/,
1725                       null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1726
1727                 input.setRd(adj.getVrfId());
1728             }
1729             if (operationalAdjacency == null) {
1730                 operationalAdjacency = populator.createOperationalAdjacency(input);
1731             }
1732             adjacencies.add(operationalAdjacency);
1733             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1734             VpnInterfaceOpDataEntry newVpnIntf =
1735                 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1736                     aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1737                         currVpnIntf.getGatewayMacAddress());
1738
1739             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1740         }
1741     }
1742
1743     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1744         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1745         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1746     }
1747
1748     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1749             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1750         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1751                                           LogicalDatastoreType.OPERATIONAL, identifier);
1752
1753         if (optVpnInterface.isPresent()) {
1754             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1755
1756             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1757             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1758             if (optAdjacencies.isPresent()) {
1759                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1760
1761                 if (!adjacencies.isEmpty()) {
1762                     LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1763                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1764                     while (adjIt.hasNext()) {
1765                         Adjacency adjElem = adjIt.next();
1766                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1767                             String rd = adjElem.getVrfId();
1768                             adjIt.remove();
1769
1770                             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1771                             VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1772                                     .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1773                                     currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove(),
1774                                             currVpnIntf.getLportTag(), currVpnIntf.getGatewayMacAddress());
1775
1776                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1777                             if (adj.getNextHopIpList() != null) {
1778                                 for (String nh : adj.getNextHopIpList()) {
1779                                     deleteExtraRouteFromCurrentAndImportingVpns(
1780                                         currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1781                                         currVpnIntf.getName(), writeConfigTxn);
1782                                 }
1783                             } else if (adj.isPhysNetworkFunc()) {
1784                                 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1785                                         adj.getIpAddress(), adj.getSubnetId());
1786                                 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1787                                         writeConfigTxn);
1788                             }
1789                             break;
1790                         }
1791
1792                     }
1793                 }
1794                 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1795                         dpnId, adj.getVrfId());
1796             } else {
1797                 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1798                         + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1799             }
1800         }
1801     }
1802
1803     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1804         String rd, String intfName, WriteTransaction writeConfigTxn) {
1805         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1806         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1807         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1808             String vpnRd = vpn.getVrfId();
1809             if (vpnRd != null) {
1810                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1811             }
1812         }
1813     }
1814
1815     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1816         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1817             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1818             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1819     }
1820
1821     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1822         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1823             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1824     }
1825
1826     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1827         WriteTransaction writeOperTxn) {
1828         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1829         if (dpId.equals(BigInteger.ZERO)) {
1830             LOG.error("addToNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1831                     + " association model", vpnInterfaceName, routerName);
1832
1833             return;
1834         }
1835         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1836
1837         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1838             .OPERATIONAL, routerDpnListIdentifier);
1839         RouterInterfaces routerInterface =
1840             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1841                 vpnInterfaceName).build();
1842         if (optionalRouterDpnList.isPresent()) {
1843             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1844                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1845         } else {
1846             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1847             builder.setRouterId(routerName);
1848             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1849             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
1850             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1851                 getRouterId(routerName),
1852                 builder.build(), true);
1853         }
1854     }
1855
1856     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1857         WriteTransaction writeOperTxn) {
1858         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1859         if (dpId.equals(BigInteger.ZERO)) {
1860             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1861                     + " dissociation model", vpnInterfaceName, routerName);
1862
1863             return;
1864         }
1865         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1866         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1867             .OPERATIONAL, routerDpnListIdentifier);
1868         if (optionalRouterDpnList.isPresent()) {
1869             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1870             RouterInterfaces routerInterface =
1871                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1872                     vpnInterfaceName).build();
1873
1874             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1875                 if (routerInterfaces.isEmpty()) {
1876                     if (writeOperTxn != null) {
1877                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1878                     } else {
1879                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1880                     }
1881                 } else {
1882                     if (writeOperTxn != null) {
1883                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1884                             RouterInterfaces.class,
1885                             new RouterInterfacesKey(vpnInterfaceName)));
1886                     } else {
1887                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1888                             routerDpnListIdentifier.child(
1889                                 RouterInterfaces.class,
1890                                 new RouterInterfacesKey(vpnInterfaceName)));
1891                     }
1892                 }
1893             }
1894         }
1895     }
1896
1897     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1898         WriteTransaction writeOperTxn) {
1899         if (dpId.equals(BigInteger.ZERO)) {
1900             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1901                     + " dissociation model", vpnInterfaceName, routerName);
1902
1903             return;
1904         }
1905         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1906         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1907             .OPERATIONAL, routerDpnListIdentifier);
1908         if (optionalRouterDpnList.isPresent()) {
1909             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1910             RouterInterfaces routerInterface =
1911                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1912                     vpnInterfaceName).build();
1913             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1914                 if (routerInterfaces.isEmpty()) {
1915                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1916                 } else {
1917                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1918                         RouterInterfaces.class,
1919                         new RouterInterfacesKey(vpnInterfaceName)));
1920                 }
1921             }
1922         }
1923     }
1924
1925     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1926                                                     WriteTransaction writeConfigTxn, String vpnName) {
1927         if (vpnInterface == null) {
1928             return;
1929         }
1930         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1931         if (adjs == null) {
1932             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1933                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1934             return;
1935         }
1936         for (Adjacency adj : adjs) {
1937             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1938                 String primaryInterfaceIp = adj.getIpAddress();
1939                 String macAddress = adj.getMacAddress();
1940                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1941
1942                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1943                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1944
1945                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1946                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1947                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1948                         routerInt, label, writeConfigTxn);
1949                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1950                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1951                         macAddress);
1952                 return;
1953             }
1954         }
1955         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1956                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1957                 primaryRd, vpnName);
1958     }
1959
1960     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1961                                                     WriteTransaction writeConfigTxn, String vpnName) {
1962         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
1963         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1964         if (adjs != null) {
1965             List<Adjacency> adjsList = adjs.getAdjacency();
1966             for (Adjacency adj : adjsList) {
1967                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1968                     String primaryInterfaceIp = adj.getIpAddress();
1969                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1970                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1971                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1972                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1973                     return;
1974                 }
1975             }
1976         } else {
1977             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1978                     vpnInterface.getName(), rd);
1979         }
1980     }
1981
1982     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1983         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1984             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1985                   vpnName, intefaceData.vpnInterface.getName());
1986             return;
1987         }
1988         final VpnInterfaceKey key = intefaceData.identifier
1989                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1990         final String interfaceName = key.getName();
1991         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1992                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1993         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1994                   intefaceData.identifier, vpnName);
1995         return;
1996     }
1997
1998     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1999                                               VpnInterface vpnInterface, String vpnName) {
2000         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2001                .get(vpnName);
2002         if (vpnInterfaces == null) {
2003             vpnInterfaces = new ConcurrentLinkedQueue<>();
2004         }
2005         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2006         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2007         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2008                 vpnInterface.getName(), vpnName);
2009     }
2010
2011     public boolean isVpnInstanceReady(String vpnInstanceName) {
2012         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2013         if (vpnRd == null) {
2014             return false;
2015         }
2016         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2017
2018         return vpnInstanceOpDataEntry != null;
2019     }
2020
2021     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2022         synchronized (vpnInstanceName.intern()) {
2023             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2024                     unprocessedVpnInterfaces.get(vpnInstanceName);
2025             if (vpnInterfaces != null) {
2026                 while (!vpnInterfaces.isEmpty()) {
2027                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2028                     if (hasVpnInstanceCreatedSuccessfully) {
2029                         processSavedInterface(savedInterface, vpnInstanceName);
2030                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2031                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2032                     } else {
2033                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2034                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2035                     }
2036                 }
2037             } else {
2038                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2039             }
2040         }
2041     }
2042
2043     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2044             VpnInterface vpnInterface) {
2045         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2046             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2047                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2048             if (vpnInterfaces != null) {
2049                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2050                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2051                             + "unprocessed list", vpnInterface.getName(),
2052                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2053                 }
2054             } else {
2055                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2056                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2057             }
2058         }
2059     }
2060
2061     public void vpnInstanceIsReady(String vpnInstanceName) {
2062         processSavedInterfaces(vpnInstanceName, true);
2063     }
2064
2065     public void vpnInstanceFailed(String vpnInstanceName) {
2066         processSavedInterfaces(vpnInstanceName, false);
2067     }
2068
2069     private static class UnprocessedVpnInterfaceData {
2070         InstanceIdentifier<VpnInterface> identifier;
2071         VpnInterface vpnInterface;
2072
2073         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2074             this.identifier = identifier;
2075             this.vpnInterface = vpnInterface;
2076         }
2077
2078         @Override
2079         public int hashCode() {
2080             final int prime = 31;
2081             int result = 1;
2082             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2083             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2084             return result;
2085         }
2086
2087         @Override
2088         public boolean equals(Object obj) {
2089             if (this == obj) {
2090                 return true;
2091             }
2092             if (obj == null) {
2093                 return false;
2094             }
2095             if (getClass() != obj.getClass()) {
2096                 return false;
2097             }
2098             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2099             if (identifier == null) {
2100                 if (other.identifier != null) {
2101                     return false;
2102                 }
2103             } else if (!identifier.equals(other.identifier)) {
2104                 return false;
2105             }
2106             if (vpnInterface == null) {
2107                 if (other.vpnInterface != null) {
2108                     return false;
2109                 }
2110             } else if (!vpnInterface.equals(other.vpnInterface)) {
2111                 return false;
2112             }
2113             return true;
2114         }
2115     }
2116
2117     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2118         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2119         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2120         if (vpnInstanceOpData == null) {
2121             return;
2122         }
2123         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2124         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2125             return;
2126         }
2127         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2128         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2129             InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2130                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2131             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = VpnUtil.read(dataBroker,
2132                     LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2133             if (!vpnInterfaceOptional.isPresent()) {
2134                 return;
2135             }
2136             List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2137                     vpnInterface.getInterfaceName());
2138             if (configVpnAdjacencies == null) {
2139                 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2140                 return;
2141             }
2142             List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2143                     .getAugmentation(AdjacenciesOp.class).getAdjacency();
2144             // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2145             // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2146             // in operational DS. These unprocessed adjacencies will be handled below.
2147             // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2148             // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2149             configVpnAdjacencies.stream()
2150                     .filter(adjacency -> operationVpnAdjacencies.stream()
2151                             .noneMatch(operationalAdjacency ->
2152                                     operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2153                     .forEach(adjacency -> {
2154                         LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2155                         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2156                             () -> {
2157                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
2158                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
2159                                 if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2160                                     addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd, adjacency,
2161                                             vpnInterfaceOptional.get().getDpnId(), writeConfigTxn, writeOperTxn);
2162                                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
2163                                     try {
2164                                         operFuture.get();
2165                                     } catch (ExecutionException | InterruptedException e) {
2166                                         LOG.error("Exception encountered while submitting operational"
2167                                                 + " future for vpnInterface {}", vpnInterface, e);
2168                                     }
2169                                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2170                                     futures.add(writeConfigTxn.submit());
2171                                     return futures;
2172                                 }  else {
2173                                     return Collections.emptyList();
2174                                 }
2175                             });
2176                     });
2177         }));
2178     }
2179
2180     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2181         private final String interfaceName;
2182         private final boolean add;
2183         private final String txnDestination;
2184
2185         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2186             this.interfaceName = interfaceName;
2187             this.add = add;
2188             this.txnDestination = transactionDest;
2189         }
2190
2191         @Override
2192         public void onSuccess(Void voidObj) {
2193             if (add) {
2194                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2195                         interfaceName, txnDestination);
2196             } else {
2197                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2198             }
2199         }
2200
2201         @Override
2202         public void onFailure(Throwable throwable) {
2203             if (add) {
2204                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2205                         interfaceName, txnDestination, throwable);
2206             } else {
2207                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2208                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
2209             }
2210         }
2211     }
2212 }