vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.concurrent.ConcurrentHashMap;
  25 import java.util.concurrent.ConcurrentLinkedQueue;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34
  35 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  36 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  37 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  41 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  42 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  43 import org.opendaylight.genius.mdsalutil.NWUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.cache.DataObjectCache;
  46 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  47 import org.opendaylight.infrautils.caches.CacheProvider;
  48 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  49 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  50 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  51 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  52 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  53 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  54 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  55 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  56 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  57 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  58 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  59 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  60 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 110 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 111 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 112 import org.slf4j.Logger;
 113 import org.slf4j.LoggerFactory;
 114
 115 @Singleton
 116 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 117
 118     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 119     private static final short DJC_MAX_RETRIES = 3;
 120
 121     private final DataBroker dataBroker;
 122     private final ManagedNewTransactionRunner txRunner;
 123     private final IBgpManager bgpManager;
 124     private final IFibManager fibManager;
 125     private final IMdsalApiManager mdsalManager;
 126     private final IdManagerService idManager;
 127     private final OdlInterfaceRpcService ifaceMgrRpcService;
 128     private final VpnFootprintService vpnFootprintService;
 129     private final IInterfaceManager interfaceManager;
 130     private final IVpnManager vpnManager;
 131     private final ArpResponderHandler arpResponderHandler;
 132     private final JobCoordinator jobCoordinator;
 133
 134     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 135
 136     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 137             new ConcurrentHashMap<>();
 138
 139     private final DataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 140
 141     @Inject
 142     public VpnInterfaceManager(final DataBroker dataBroker,
 143                                final IBgpManager bgpManager,
 144                                final IdManagerService idManager,
 145                                final IMdsalApiManager mdsalManager,
 146                                final IFibManager fibManager,
 147                                final OdlInterfaceRpcService ifaceMgrRpcService,
 148                                final VpnFootprintService vpnFootprintService,
 149                                final IInterfaceManager interfaceManager,
 150                                final IVpnManager vpnManager,
 151                                final ArpResponderHandler arpResponderHandler,
 152                                final JobCoordinator jobCoordinator,
 153                                final CacheProvider cacheProvider) {
 154         super(VpnInterface.class, VpnInterfaceManager.class);
 155
 156         this.dataBroker = dataBroker;
 157         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 158         this.bgpManager = bgpManager;
 159         this.idManager = idManager;
 160         this.mdsalManager = mdsalManager;
 161         this.fibManager = fibManager;
 162         this.ifaceMgrRpcService = ifaceMgrRpcService;
 163         this.vpnFootprintService = vpnFootprintService;
 164         this.interfaceManager = interfaceManager;
 165         this.vpnManager = vpnManager;
 166         this.arpResponderHandler = arpResponderHandler;
 167         this.jobCoordinator = jobCoordinator;
 168
 169         vpnInstanceOpDataEntryCache = new DataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 170                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 171                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 172     }
 173
 174     public Runnable isNotifyTaskQueued(String intfName) {
 175         return vpnIntfMap.remove(intfName);
 176     }
 177
 178     @PostConstruct
 179     public void start() {
 180         LOG.info("{} start", getClass().getSimpleName());
 181         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 182     }
 183
 184     @Override
 185     @PreDestroy
 186     public void close() {
 187         super.close();
 188         vpnInstanceOpDataEntryCache.close();
 189     }
 190
 191     @Override
 192     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 193         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 194     }
 195
 196     @Override
 197     protected VpnInterfaceManager getDataTreeChangeListener() {
 198         return VpnInterfaceManager.this;
 199     }
 200
 201     @Override
 202     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 203         LOG.info("add: intfName {} onto vpnName {}",
 204                  vpnInterface.getName(),
 205                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 206         addVpnInterface(identifier, vpnInterface, null, null);
 207     }
 208
 209     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 210                           final VpnInterface vpnInterface, String vpnName) {
 211         synchronized (vpnName.intern()) {
 212             if (isVpnInstanceReady(vpnName)) {
 213                 return true;
 214             }
 215             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 216             return false;
 217         }
 218     }
 219
 220     // TODO Clean up the exception handling
 221     @SuppressWarnings("checkstyle:IllegalCatch")
 222     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 223                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 224         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 225             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 226             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 227         }
 228     }
 229
 230     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 231                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 232         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 233         final String interfaceName = key.getName();
 234
 235         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 236             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 237                   vpnName, vpnInterface.getName());
 238             return;
 239         }
 240         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 241              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 242         List<Adjacency> copyOldAdjs = null;
 243         if (oldAdjs != null) {
 244             copyOldAdjs = new ArrayList<>();
 245             copyOldAdjs.addAll(oldAdjs);
 246         }
 247         List<Adjacency> copyNewAdjs = null;
 248         if (newAdjs != null) {
 249             copyNewAdjs = new ArrayList<>();
 250             copyNewAdjs.addAll(newAdjs);
 251         }
 252         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 253     }
 254
 255     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 256                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 257                     final List<Adjacency> newAdjs,
 258                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 259         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 260         final String interfaceName = key.getName();
 261         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 262         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 263             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 264             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 265             if (interfaceState != null) {
 266                 try {
 267                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 268                     final int ifIndex = interfaceState.getIfIndex();
 269                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 270                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 271                         WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 272                         WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 273                         LOG.info("addVpnInterface: VPN Interface add event - intfName {} vpnName {} on dpn {}" ,
 274                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 275                         processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, writeConfigTxn,
 276                                      writeOperTxn, writeInvTxn, interfaceState, vpnName);
 277                         if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 278                             LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 279                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 280                             if (newAdjs != null) {
 281                                 for (Adjacency adj : newAdjs) {
 282                                     if (oldAdjs.contains(adj)) {
 283                                         oldAdjs.remove(adj);
 284                                     } else {
 285                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 286                                                                   dataBroker, adj)) {
 287                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd,
 288                                                   adj, dpnId, writeOperTxn, writeConfigTxn);
 289                                         }
 290                                     }
 291                                 }
 292                             }
 293                             for (Adjacency adj : oldAdjs) {
 294                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 295                                                           dataBroker, adj)) {
 296                                     delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 297                                         writeOperTxn, writeConfigTxn);
 298                                 }
 299                             }
 300                         }
 301                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
 302                         try {
 303                             operFuture.get();
 304                         } catch (ExecutionException e) {
 305                             LOG.error("addVpnInterface: Exception encountered while submitting operational future for"
 306                                     + " addVpnInterface {} on vpn {}: {}", vpnInterface.getName(), vpnName, e);
 307                             return null;
 308                         }
 309                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 310                         ListenableFuture<Void> configFuture = writeConfigTxn.submit();
 311                         futures.add(configFuture);
 312                         Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"));
 313                         futures.add(writeInvTxn.submit());
 314                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 315                                 + " processed successfully", interfaceName, vpnName, dpnId);
 316                         return futures;
 317                     });
 318                 } catch (NumberFormatException | IllegalStateException e) {
 319                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 320                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 321                                     vpnName, e);
 322                     return;
 323                 }
 324             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 325                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 326                     () -> {
 327                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 328                         createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 329                                        writeConfigTxn, vpnName);
 330                         LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 331                                  vpnName, vpnInterface.getDpnId());
 332                         ListenableFuture<Void> futures = writeConfigTxn.submit();
 333                         String errorText = "addVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
 334                              + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
 335                         ListenableFutures.addErrorLogging(futures, LOG, errorText);
 336                         return Collections.singletonList(futures);
 337                     });
 338             } else {
 339                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 340                         + " is not available", interfaceName, vpnName);
 341             }
 342         } else {
 343             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 344                       + " as vpn is pending delete", interfaceName, vpnName,
 345                     vpnInterface.getDpnId());
 346         }
 347     }
 348
 349     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 350     // see comments below.
 351     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 352     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 353             final int lportTag, boolean isInterfaceUp,
 354             WriteTransaction writeConfigTxn,
 355             WriteTransaction writeOperTxn,
 356             WriteTransaction writeInvTxn,
 357             Interface interfaceState,
 358             final String vpnName) {
 359         final String interfaceName = vpnInterface.getName();
 360         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 361                                                          interfaceName, vpnName);
 362         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 363         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 364         if (!isInterfaceUp) {
 365             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 366                      interfaceName, dpId, vpnName);
 367             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 368             if (vpnId == VpnConstants.INVALID_ID) {
 369                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 370                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 371                         dpId);
 372                 return;
 373             }
 374
 375             boolean waitForVpnInterfaceOpRemoval = false;
 376             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 377                 String opVpnName = opVpnInterface.getVpnInstanceName();
 378                 String primaryInterfaceIp = null;
 379                 if (opVpnName.equals(vpnName)) {
 380                     // Please check if the primary VRF Entry does not exist for VPNInterface
 381                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 382                     // back to back
 383                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 384                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 385                     if (adjs == null) {
 386                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 387                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 388                                 vpnName);
 389                         return;
 390                     }
 391                     for (Adjacency adj : adjs) {
 392                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 393                             primaryInterfaceIp = adj.getIpAddress();
 394                             break;
 395                         }
 396                     }
 397                     if (primaryInterfaceIp == null) {
 398                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 399                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 400                                 dpId, vpnName);
 401                         return;
 402                     }
 403                     // Get the rd of the vpn instance
 404                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 405                     if (vrf != null) {
 406                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 407                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 408                         return;
 409                     }
 410                     waitForVpnInterfaceOpRemoval = true;
 411                 } else {
 412                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 413                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 414                 }
 415             }
 416             if (!waitForVpnInterfaceOpRemoval) {
 417                 // Add the VPNInterface and quit
 418                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 419                         null/*ipAddressSourceValuePair*/,
 420                         true /* add */);
 421                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 422                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 423                 if (!isBgpVpnInternetVpn) {
 424                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 425                             jobCoordinator);
 426                 }
 427                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 428                         dpId, vpnName);
 429                 if (interfaceManager.isExternalInterface(interfaceName)) {
 430                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 431                         NwConstants.ADD_FLOW);
 432                 }
 433                 return;
 434             }
 435
 436             // FIB didn't get a chance yet to clean up this VPNInterface
 437             // Let us give it a chance here !
 438             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 439                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 440             try {
 441                 Runnable notifyTask = new VpnNotifyTask();
 442                 synchronized (notifyTask) {
 443                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 444                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 445                     // published here while holding the lock on it so this path will hit the wait before notify can be
 446                     // invoked.
 447                     vpnIntfMap.put(interfaceName, notifyTask);
 448                     try {
 449                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 450                     } catch (InterruptedException e) {
 451                         // Ignored
 452                     }
 453                 }
 454             } finally {
 455                 vpnIntfMap.remove(interfaceName);
 456             }
 457
 458             if (opVpnInterface != null) {
 459                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 460                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 461                         dpId, vpnName);
 462                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
 463                 return;
 464             }
 465             // VPNInterface got removed, proceed with Add
 466             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 467                     interfaceName, dpId, vpnName);
 468             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 469                     null/*ipAddressSourceValuePair*/,
 470                     true /* add */);
 471             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 472                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 473             if (!isBgpVpnInternetVpn) {
 474                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 475             }
 476             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 477                     + " FIB to clean up", interfaceName, dpId, vpnName);
 478             if (interfaceManager.isExternalInterface(interfaceName)) {
 479                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 480                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 481             }
 482         } else {
 483             // Interface is retained in the DPN, but its Link Up.
 484             // Advertise prefixes again for this interface to BGP
 485             InstanceIdentifier<VpnInterface> identifier =
 486                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 487             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 488                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 489             advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 490             // Perform similar operation as interface add event for extraroutes.
 491             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 492             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 493             if (!optAdjacencies.isPresent()) {
 494                 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 495                 return;
 496             }
 497             List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 498             for (Adjacency adjacency : adjacencies) {
 499                 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 500                     continue;
 501                 }
 502                 // if BGPVPN Internet, filter only IPv6 Adjacencies
 503                 if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 504                                 dataBroker, adjacency)) {
 505                     continue;
 506                 }
 507                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 508                           dpId, writeOperTxn, writeConfigTxn);
 509             }
 510         }
 511     }
 512
 513     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 514             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 515         Uuid extNetworkId;
 516         try {
 517             // vpn instance of ext-net interface is the network-id
 518             extNetworkId = new Uuid(vpnName);
 519         } catch (IllegalArgumentException e) {
 520             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 521                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 522             return;
 523         }
 524
 525         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 526         if (routerIds == null || routerIds.isEmpty()) {
 527             LOG.info("processExternalVpnInterface: No router is associated with {}."
 528                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 529                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 530             return;
 531         }
 532
 533         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 534                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 535         for (Uuid routerId : routerIds) {
 536             String routerName = routerId.getValue();
 537             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 538             if (Objects.equals(primarySwitch, dpId)) {
 539                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 540                 if (router != null) {
 541                     if (addOrRemove == NwConstants.ADD_FLOW) {
 542                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 543                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 544                                 dpId, interfaceName, lportTag, writeInvTxn);
 545                     } else {
 546                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 547                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 548                                 dpId, interfaceName, lportTag);
 549                     }
 550                 } else {
 551                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 552                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 553                             interfaceName, dpId, vpnName);
 554                 }
 555             }
 556         }
 557     }
 558
 559     // TODO Clean up the exception handling
 560     @SuppressWarnings("checkstyle:IllegalCatch")
 561     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 562                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 563                                                   String vpnName, String interfaceName) {
 564         if (rd == null) {
 565             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 566                   interfaceName, dpnId, vpnName);
 567             return;
 568         } else {
 569             if (rd.equals(vpnName)) {
 570                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 571                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 572                 return;
 573             }
 574         }
 575         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 576                 interfaceName, dpnId, vpnName, rd);
 577
 578         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 579         if (nextHopIp == null) {
 580             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 581                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 582                     rd, vpnName);
 583             return;
 584         }
 585
 586         //Read NextHops
 587         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 588         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 589         if (adjacencies.isPresent()) {
 590             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 591
 592             if (!nextHops.isEmpty()) {
 593                 LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 594                         + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 595                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 596                 long l3vni = vpnInstanceOpData.getL3vni();
 597                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 598                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 599                 for (Adjacency nextHop : nextHops) {
 600                     if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 601                         continue;
 602                     }
 603                     String gatewayMac = null;
 604                     long label = 0;
 605                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 606                         final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 607                             vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 608                         gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName).get();
 609                     } else {
 610                         label = nextHop.getLabel();
 611                     }
 612                     try {
 613                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 614                                 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 615                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 616                                 encapType, (int)label, l3vni, 0 /*l2vni*/,
 617                                 gatewayMac);
 618                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 619                                 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 620                                 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 621                     } catch (Exception e) {
 622                         LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}"
 623                                 + " for interface {} on dpn {}", nextHop.getIpAddress(),
 624                                   vpnName, rd, interfaceName, dpnId, e);
 625                     }
 626                 }
 627             }
 628         }
 629     }
 630
 631     // TODO Clean up the exception handling
 632     @SuppressWarnings("checkstyle:IllegalCatch")
 633     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 634                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 635         //Read NextHops
 636         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 637         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 638
 639         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 640         if (rd == null) {
 641             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 642                 interfaceName, vpnName);
 643             return;
 644         } else {
 645             if (rd.equals(vpnName)) {
 646                 LOG.info(
 647                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 648                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 649                 return;
 650             }
 651         }
 652         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 653                vpnName, rd);
 654         if (adjacencies.isPresent()) {
 655             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 656
 657             if (!nextHops.isEmpty()) {
 658                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 659                         nextHops, interfaceName, vpnName, rd);
 660                 for (Adjacency nextHop : nextHops) {
 661                     try {
 662                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 663                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 664                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 665                                     interfaceName, vpnName);
 666                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 667                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 668                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 669                                     interfaceName, vpnName);
 670                         } else {
 671                             // Perform similar operation as interface delete event for extraroutes.
 672                             String allocatedRd = nextHop.getVrfId();
 673                             for (String nh : nextHop.getNextHopIpList()) {
 674                                 deleteExtraRouteFromCurrentAndImportingVpns(
 675                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 676                             }
 677                         }
 678                     } catch (Exception e) {
 679                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 680                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 681                     }
 682                 }
 683             }
 684         }
 685     }
 686
 687     @SuppressWarnings("checkstyle:IllegalCatch")
 688     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 689                                                   String primaryRd, String interfaceName, final long vpnId,
 690                                                   WriteTransaction writeConfigTxn,
 691                                                   WriteTransaction writeOperTxn,
 692                                                   final WriteTransaction writeInvTxn,
 693                                                   Interface interfaceState) {
 694         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 695         // Read NextHops
 696         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 697         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 698         if (!adjacencies.isPresent()) {
 699             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 700                     null/*gwMac*/, writeOperTxn);
 701             return;
 702         }
 703
 704         // Get the rd of the vpn instance
 705         String nextHopIp = null;
 706         try {
 707             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 708         } catch (Exception e) {
 709             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 710                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 711         }
 712         List<String> nhList = new ArrayList<>();
 713         if (nextHopIp != null) {
 714             nhList.add(nextHopIp);
 715             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 716                     interfaceName, dpnId, vpnName, nhList);
 717         }
 718         Optional<String> gwMac = Optional.absent();
 719         String vpnInterfaceSubnetGwMacAddress = null;
 720         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 721         Long l3vni = vpnInstanceOpData.getL3vni();
 722         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 723         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 724         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 725         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 726         List<Adjacency> value = new ArrayList<>();
 727         for (Adjacency nextHop : nextHops) {
 728             String rd = primaryRd;
 729             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 730             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 731                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 732                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 733                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 734                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 735                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 736                 continue;
 737             }
 738             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 739                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 740                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 741                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 742                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 743                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 744                 writeOperTxn.merge(
 745                     LogicalDatastoreType.OPERATIONAL,
 746                     VpnUtil.getPrefixToInterfaceIdentifier(
 747                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 748                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 749                             prefixCue), true);
 750                 final Uuid subnetId = nextHop.getSubnetId();
 751
 752                 String gatewayIp = nextHop.getSubnetGatewayIp();
 753                 if (gatewayIp == null) {
 754                     Optional<String> gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 755                     if (gatewayIpOptional.isPresent()) {
 756                         gatewayIp = gatewayIpOptional.get();
 757                     }
 758                 }
 759
 760                 if (gatewayIp != null) {
 761                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 762                     if (gwMac.isPresent()) {
 763                         // A valid mac-address is available for this subnet-gateway-ip
 764                         // Use this for programming ARP_RESPONDER table here.  And save this
 765                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 766                         // to populate L3_GW_MAC_TABLE there.
 767                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 768                                 gatewayIp, gwMac.get());
 769                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 770                     } else {
 771                         // A valid mac-address is not available for this subnet-gateway-ip
 772                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 773                         // Save this connected-mac-address as gateway-mac-address for the
 774                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 775                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 776                         if (gwMac.isPresent()) {
 777                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 778                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 779                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 780                                     gatewayIp, gwMac.get());
 781                         } else {
 782                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 783                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 784                                 + "gwIp {}",
 785                                 interfaceName, vpnName, gatewayIp);
 786                         }
 787                     }
 788                 } else {
 789                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 790                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 791                         subnetId, interfaceName, vpnName);
 792                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 793                 }
 794                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 795                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 796             } else {
 797                 //Extra route adjacency
 798                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 799                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 800                 synchronized (vpnPrefixKey.intern()) {
 801                     java.util.Optional<String> rdToAllocate = VpnUtil
 802                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 803                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 804                     if (rdToAllocate.isPresent()) {
 805                         rd = rdToAllocate.get();
 806                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 807                             rd, prefix);
 808                     } else {
 809                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 810                         continue;
 811                     }
 812                 }
 813                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 814                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 815                         interfaceName, dpnId);
 816             }
 817             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 818             // can be different from the gateway-mac-address within the VRFEntry as the
 819             // gateway-mac-address is a superset.
 820             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 821                     : RouteOrigin.STATIC;
 822             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 823                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 824                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 825             Adjacency operationalAdjacency = null;
 826             try {
 827                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 828             } catch (NullPointerException e) {
 829                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 830                     input, e.getMessage());
 831                 return;
 832             }
 833             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 834                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 835                         vpnName, l3vni, origin,
 836                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 837             }
 838             value.add(operationalAdjacency);
 839         }
 840
 841         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 842         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 843                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 844
 845         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 846                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 847                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 848
 849         for (Adjacency nextHop : aug.getAdjacency()) {
 850             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 851             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 852                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 853                         : RouteOrigin.STATIC;
 854                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 855                 registeredPopulator.populateFib(input, writeConfigTxn);
 856             }
 857         }
 858     }
 859
 860     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 861                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 862         VpnInterfaceOpDataEntry opInterface =
 863               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 864         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 865             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 866         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 867                 WriteTransaction.CREATE_MISSING_PARENTS);
 868         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 869                 interfaceName, dpnId, vpnName);
 870     }
 871
 872     // TODO Clean up the exception handling
 873     @SuppressWarnings("checkstyle:IllegalCatch")
 874     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 875                                            StateTunnelList stateTunnelList,
 876                                            WriteTransaction writeConfigTxn,
 877                                            WriteTransaction writeOperTxn) {
 878
 879         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 880         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 881         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 882         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 883         if (adjList.isEmpty()) {
 884             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 885                     vpnInterface, srcDpnId);
 886             return;
 887         }
 888         String prefix = null;
 889         long label = 0;
 890         List<Adjacency> value = new ArrayList<>();
 891         boolean isNextHopAddReqd = false;
 892         String vpnName = vpnInterface.getVpnInstanceName();
 893         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 894         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 895         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 896                 vpnInterface.getName(), vpnInterface.getDpnId(),
 897                 vpnInterface.getVpnInstanceName(), adjList);
 898         for (Adjacency adj : adjList) {
 899             String rd = adj.getVrfId();
 900             rd = rd != null ? rd : vpnName;
 901             prefix = adj.getIpAddress();
 902             label = adj.getLabel();
 903             List<String> nhList = Collections.singletonList(srcTepIp);
 904             List<String> nextHopList = adj.getNextHopIpList();
 905             // If TEP is added , update the nexthop of primary adjacency.
 906             // Secondary adj nexthop is already pointing to primary adj IP address.
 907             if (nextHopList != null && !nextHopList.isEmpty() && nextHopList.get(0).equalsIgnoreCase(srcTepIp)) {
 908                 /* everything right already */
 909             } else {
 910                 isNextHopAddReqd = true;
 911             }
 912
 913             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 914                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 915             } else {
 916                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 917                 if (!vrfEntryOptional.isPresent()) {
 918                     continue;
 919                 }
 920                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 921                 if (!nhList.contains(srcTepIp)) {
 922                     nhList.add(srcTepIp);
 923                     isNextHopAddReqd = true;
 924                 }
 925                 value.add(adj);
 926             }
 927
 928             if (isNextHopAddReqd) {
 929                 updateLabelMapper(label, nhList);
 930                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 931                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 932                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 933                 // Update the VRF entry with nextHop
 934                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 935                         label, true, writeConfigTxn);
 936
 937                 //Get the list of VPN's importing this route(prefix) .
 938                 // Then update the VRF entry with nhList
 939                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 940                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 941                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 942                     String vpnRd = vpn.getVrfId();
 943                     if (vpnRd != null) {
 944                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 945                                 srcTepIp, label, true, writeConfigTxn);
 946                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 947                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 948                             vpnInterface.getName(), srcDpnId, vpnName,
 949                             vpn.getVpnInstanceName(), vpnRd);
 950                     }
 951                 }
 952                 // Advertise the prefix to BGP only for external vpn
 953                 // since there is a nexthop change.
 954                 try {
 955                     if (!rd.equalsIgnoreCase(vpnName)) {
 956                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 957                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 958                                 null /*gatewayMacAddress*/);
 959                     }
 960                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 961                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 962                             srcDpnId, vpnName);
 963                 } catch (Exception ex) {
 964                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 965                             + " on rd {} for interface {} on dpn {} vpn {} as {}", prefix, nhList, label, rd,
 966                             vpnInterface.getName(), srcDpnId,
 967                             vpnName, ex);
 968                 }
 969             }
 970         }
 971         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 972         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 973                 .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 974                 .addAugmentation(AdjacenciesOp.class, aug).build();
 975         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 976                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 977         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 978                 WriteTransaction.CREATE_MISSING_PARENTS);
 979         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 980                 vpnInterface.getName(), srcDpnId, vpnName);
 981
 982     }
 983
 984     // TODO Clean up the exception handling
 985     @SuppressWarnings("checkstyle:IllegalCatch")
 986     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
 987                                               StateTunnelList stateTunnelList,
 988                                               WriteTransaction writeConfigTxn,
 989                                               WriteTransaction writeOperTxn) {
 990
 991         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 992         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 993         String prefix = null;
 994         long label = 0;
 995         boolean isNextHopRemoveReqd = false;
 996         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 997         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 998         String vpnName = vpnInterface.getVpnInstanceName();
 999         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1000         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1001         if (adjList != null) {
1002             List<Adjacency> value = new ArrayList<>();
1003             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1004                     vpnInterface.getName(), vpnInterface.getDpnId(),
1005                     vpnInterface.getVpnInstanceName(), adjList);
1006             for (Adjacency adj : adjList) {
1007                 List<String> nhList = new ArrayList<>();
1008                 String rd = adj.getVrfId();
1009                 rd = rd != null ? rd : vpnName;
1010                 prefix = adj.getIpAddress();
1011                 List<String> nextHopList = adj.getNextHopIpList();
1012                 label = adj.getLabel();
1013                 if (nextHopList != null && !nextHopList.isEmpty()) {
1014                     isNextHopRemoveReqd = true;
1015                 }
1016                 // If TEP is deleted , remove the nexthop from primary adjacency.
1017                 // Secondary adj nexthop will continue to point to primary adj IP address.
1018                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1019                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1020                 } else {
1021                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1022                     if (!vrfEntryOptional.isPresent()) {
1023                         continue;
1024                     }
1025                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1026                     if (nhList.contains(srcTepIp)) {
1027                         nhList.remove(srcTepIp);
1028                         isNextHopRemoveReqd = true;
1029                     }
1030                     value.add(adj);
1031                 }
1032
1033                 if (isNextHopRemoveReqd) {
1034                     updateLabelMapper(label, nhList);
1035                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1036                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1037                             prefix, nhList, vpnName,
1038                             vpnId, rd, vpnInterface.getName());
1039                     // Update the VRF entry with removed nextHop
1040                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1041                             label, false, writeConfigTxn);
1042
1043                     //Get the list of VPN's importing this route(prefix) .
1044                     // Then update the VRF entry with nhList
1045                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1046                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1047                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1048                         String vpnRd = vpn.getVrfId();
1049                         if (vpnRd != null) {
1050                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1051                                     srcTepIp, label, false, writeConfigTxn);
1052                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1053                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1054                                     nhList, label, vpnInterface.getName(), srcDpnId,
1055                                     vpnName,
1056                                     vpn.getVpnInstanceName(), vpnRd);
1057                         }
1058                     }
1059
1060                     // Withdraw prefix from BGP only for external vpn.
1061                     try {
1062                         if (!rd.equalsIgnoreCase(vpnName)) {
1063                             bgpManager.withdrawPrefix(rd, prefix);
1064                         }
1065                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1066                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1067                                 vpnInterface.getName(), srcDpnId,
1068                                 vpnName);
1069                     } catch (Exception ex) {
1070                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1071                                 + " on rd {} for interface {} on dpn {} vpn {} as {}", prefix, nhList, label, rd,
1072                                 vpnInterface.getName(), srcDpnId,
1073                                 vpnName, ex);
1074                     }
1075                 }
1076             }
1077             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1078             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1079                     .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1080                     .addAugmentation(AdjacenciesOp.class, aug).build();
1081             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1082                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1083             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1084                     WriteTransaction.CREATE_MISSING_PARENTS);
1085             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1086                          vpnInterface.getName(), srcDpnId, vpnName);
1087         }
1088     }
1089
1090     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1091         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1092
1093         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1094         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1095         if (vpnInstanceOpDataEntry == null) {
1096             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1097                     + " to check for vpns exporting the routes", vpnName);
1098             return vpnsToExportRoute;
1099         }
1100
1101         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1102             if (input.getVpnInstanceName() == null) {
1103                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1104                         input.getVrfId());
1105                 return false;
1106             }
1107             return !input.getVpnInstanceName().equals(vpnName);
1108         };
1109
1110         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1111             Iterable<String> commonRTs =
1112                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1113                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1114             return Iterators.size(commonRTs.iterator()) > 0;
1115         };
1116
1117         vpnsToExportRoute =
1118                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1119                         Collectors.toList());
1120         return vpnsToExportRoute;
1121     }
1122
1123     // TODO Clean up the exception handling
1124     @SuppressWarnings("checkstyle:IllegalCatch")
1125     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1126         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1127         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1128             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1129             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1130             if (vrfEntries != null) {
1131                 for (VrfEntry vrfEntry : vrfEntries) {
1132                     try {
1133                         if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1134                                 RouteOrigin.value(vrfEntry.getOrigin()))) {
1135                             LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1136                                     + " is not a controller managed non intervpn link route. Ignoring.",
1137                                     vpn.getVrfId(), vrfEntry.getDestPrefix());
1138                             continue;
1139                         }
1140                         String prefix = vrfEntry.getDestPrefix();
1141                         String gwMac = vrfEntry.getGatewayMacAddress();
1142                         vrfEntry.getRoutePaths().forEach(routePath -> {
1143                             String nh = routePath.getNexthopAddress();
1144                             int label = routePath.getLabel().intValue();
1145                             if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1146                                     vrfEntry.getOrigin()))) {
1147                                 LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1148                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1149                                         vpnName, vpnRd);
1150                                 fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1151                                         Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1152                                         0 /*l3vni*/, gwMac,  null /*parentVpnRd*/, RouteOrigin.SELF_IMPORTED,
1153                                         writeConfigTxn);
1154                             } else {
1155                                 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} prefix {}"
1156                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1157                                         vpnName, vpnRd);
1158                                 SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1159                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, writeConfigTxn);
1160                             }
1161                         });
1162                     } catch (RuntimeException e) {
1163                         LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1164                                 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1165                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1166                     }
1167                 }
1168                 writeConfigTxn.submit();
1169             } else {
1170                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1171                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1172             }
1173         }
1174     }
1175
1176     @SuppressWarnings("checkstyle:IllegalCatch")
1177     @Override
1178     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1179         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1180         final String interfaceName = key.getName();
1181         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1182             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1183             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1184         }
1185     }
1186
1187     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1188                                 final VpnInterface vpnInterface, final String vpnName,
1189                                 final String interfaceName) {
1190         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1191             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1192                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1193                 deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn, vpnName);
1194                 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1195                 ListenableFuture<Void> futures = writeConfigTxn.submit();
1196                 String errorText = "removeVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
1197                         + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
1198                 ListenableFutures.addErrorLogging(futures, LOG, errorText);
1199                 return Collections.singletonList(futures);
1200             }, DJC_MAX_RETRIES);
1201         } else {
1202             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1203             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1204         }
1205     }
1206
1207     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1208     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1209                                 final VpnInterface vpnInterface, final String vpnName,
1210                                 final String interfaceName, final Interface interfaceState) {
1211         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1212                 vpnName, vpnInterface.getDpnId());
1213         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1214         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1215             () -> {
1216                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1217                 ListenableFuture<Void> configFuture = txRunner
1218                         .callWithNewWriteOnlyTransactionAndSubmit(writeConfigTxn -> {
1219                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeOperTxn -> {
1220                                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1221                                     LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1222                                             interfaceName, vpnName);
1223                                     BigInteger dpId = BigInteger.ZERO;
1224                                     int ifIndex = 0;
1225                                     String gwMacAddress = null;
1226                                     InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1227                                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1228                                     final Optional<VpnInterfaceOpDataEntry> optVpnInterface =
1229                                             VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1230                                     if (interfaceState != null) {
1231                                         try {
1232                                             dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1233                                         } catch (NumberFormatException | IllegalStateException e) {
1234                                             LOG.error("remove: Unable to retrieve dpnId from interface operational"
1235                                                     + " data store for interface {} on dpn {} for vpn {} Fetching"
1236                                                             + " from vpn interface op data store. ", interfaceName,
1237                                                     vpnInterface.getDpnId(), vpnName, e);
1238                                             dpId = BigInteger.ZERO;
1239                                         }
1240                                         ifIndex = interfaceState.getIfIndex();
1241                                         gwMacAddress = interfaceState.getPhysAddress().getValue();
1242                                     } else {
1243                                         LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1244                                                 + " from vpn interface op.", interfaceName);
1245                                         if (optVpnInterface.isPresent()) {
1246                                             VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1247                                             dpId = vpnOpInterface.getDpnId();
1248                                             ifIndex = vpnOpInterface.getLportTag().intValue();
1249                                             gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1250                                         } else {
1251                                             LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1252                                                             + " as interfaceState and vpn interface op is not"
1253                                                     + " available", interfaceName, vpnName);
1254                                             return;
1255                                         }
1256                                     }
1257                                     processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1258                                             optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1259                                             writeConfigTxn, writeOperTxn, writeInvTxn);
1260                                     LOG.info(
1261                                             "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1262                                                     + "successfully",
1263                                             interfaceName, vpnInterface.getDpnId(), vpnName);
1264                                 }));
1265                             }));
1266                         });
1267                 futures.add(configFuture);
1268                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1269                 return futures;
1270             }, DJC_MAX_RETRIES);
1271     }
1272
1273     protected void processVpnInterfaceDown(BigInteger dpId,
1274                                            String interfaceName,
1275                                            int lportTag,
1276                                            String gwMac,
1277                                            VpnInterfaceOpDataEntry vpnOpInterface,
1278                                            boolean isInterfaceStateDown,
1279                                            WriteTransaction writeConfigTxn,
1280                                            WriteTransaction writeOperTxn,
1281                                            WriteTransaction writeInvTxn) {
1282         if (vpnOpInterface == null) {
1283             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1284                     + " as it is not available in operational data store", interfaceName, dpId);
1285             return;
1286         }
1287         final String vpnName = vpnOpInterface.getVpnInstanceName();
1288         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1289                                                     interfaceName, vpnName);
1290         if (!isInterfaceStateDown) {
1291             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1292             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1293                     null);
1294             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1295             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1296                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1297             if (interfaceManager.isExternalInterface(interfaceName)) {
1298                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1299                         NwConstants.DEL_FLOW);
1300             }
1301             if (!isBgpVpnInternetVpn) {
1302                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1303             }
1304             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1305                     + " successful", interfaceName, dpId, vpnName);
1306         } else {
1307             // Interface is retained in the DPN, but its Link Down.
1308             // Only withdraw the prefixes for this interface from BGP
1309             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1310         }
1311     }
1312
1313     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1314                                           final String vpnName, final long vpnId, String gwMac,
1315                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1316                                           final WriteTransaction writeInvTxn) {
1317         //Read NextHops
1318         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1319                 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1320         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1321         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1322
1323         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1324         LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1325                 interfaceName, dpnId, vpnName, primaryRd);
1326         if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1327             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1328             LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1329                     interfaceName, dpnId, vpnName, nextHops);
1330             for (Adjacency nextHop : nextHops) {
1331                 if (nextHop.isPhysNetworkFunc()) {
1332                     LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1333                             nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1334                     fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1335                             null/*writeCfgTxn*/);
1336                 } else {
1337                     String rd = nextHop.getVrfId();
1338                     List<String> nhList;
1339                     if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1340                         nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1341                     } else {
1342                         // This is a primary adjacency
1343                         nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1344                                 : Collections.emptyList();
1345                         removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1346                                 interfaceName, writeInvTxn);
1347                     }
1348                     if (!nhList.isEmpty()) {
1349                         if (rd.equals(vpnName)) {
1350                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1351                             //remove from FIB directly
1352                             nhList.forEach((nh) -> removeAdjacencyFromInternalVpn(nextHop, vpnName,
1353                                     interfaceName, dpnId, writeConfigTxn));
1354                         } else {
1355                             removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, interfaceName,
1356                                     writeConfigTxn);
1357                         }
1358                     } else {
1359                         LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1360                                         + " interface {}", nextHop.getIpAddress(), rd,
1361                                 nextHop.getAdjacencyType().toString(), interfaceName);
1362                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
1363                         fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1364                     }
1365                 }
1366                 String ip = nextHop.getIpAddress().split("/")[0];
1367                 LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1368                 if (vpnVipToPort != null) {
1369                     VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip);
1370                     LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1371                                     + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(),
1372                             ip, dpnId, vpnName);
1373                 }
1374             }
1375         } else {
1376             // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1377             LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1378                     + " Removing it.", interfaceName, vpnName, dpnId);
1379             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1380         }
1381     }
1382
1383     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1384                                                                    String interfaceName, BigInteger dpnId,
1385                                                                    WriteTransaction writeConfigTxn) {
1386         return (nh) -> {
1387             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1388                     writeConfigTxn);
1389             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1390                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1391                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1392         };
1393     }
1394
1395     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1396                                            BigInteger dpnId, String rd, String interfaceName,
1397                                            WriteTransaction writeConfigTxn) {
1398         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1399                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1400         nhList.forEach((nh) -> {
1401             //IRT: remove routes from other vpns importing it
1402             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1403                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1404             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1405                 String vpnRd = vpn.getVrfId();
1406                 if (vpnRd != null) {
1407                     fibManager.removeOrUpdateFibEntry(vpnRd,
1408                             nextHop.getIpAddress(), nh, writeConfigTxn);
1409                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1410                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1411                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1412                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1413                 }
1414             }
1415         });
1416     }
1417
1418     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1419                                                  int lportTag, String gwMac, String interfaceName,
1420                                                  WriteTransaction writeInvTxn) {
1421         final Uuid subnetId = nextHop.getSubnetId();
1422         if (nextHop.getSubnetGatewayMacAddress() == null) {
1423             // A valid mac-address was not available for this subnet-gateway-ip
1424             // So a connected-mac-address was used for this subnet and we need
1425             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1426             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1427                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1428         }
1429         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1430                 subnetId);
1431     }
1432
1433     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1434                                                   String interfaceName) {
1435         // This is either an extra-route (or) a learned IP via subnet-route
1436         List<String> nhList = null;
1437         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1438         if (nextHopIp == null || nextHopIp.isEmpty()) {
1439             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1440                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1441                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1442                     vpnName);
1443             nhList = Collections.emptyList();
1444         } else {
1445             nhList = Collections.singletonList(nextHopIp);
1446         }
1447         return nhList;
1448     }
1449
1450     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1451         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1452         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1453         // else use connected interface
1454         if (gwPort != null && gwPort.isSubnetIp()) {
1455             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1456                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1457             return Optional.of(gwPort.getMacAddress());
1458         }
1459         return Optional.absent();
1460     }
1461
1462     @Override
1463     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1464         final VpnInterface update) {
1465         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1466                 update.getDpnId(), original.getVpnInstanceNames(),
1467                 update.getVpnInstanceNames());
1468         final String vpnInterfaceName = update.getName();
1469         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1470         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1471         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1472             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1473         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1474         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1475             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1476
1477         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1478         //handles switching between <internal VPN - external VPN>
1479         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1480             LOG.info("update: handled VPNInterface {} on dpn {} update"
1481                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1482                      original.getName(), dpnId,
1483                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1484                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1485             return;
1486         }
1487         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1488             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1489             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1490             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1491             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1492             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1493                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1494                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1495                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1496                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1497                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1498                     LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1499                             update.getName(), newVpnName);
1500                     //handle both addition and removal of adjacencies
1501                     //currently, new adjacency may be an extra route
1502                     boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1503                     if (!oldAdjs.equals(newAdjs)) {
1504                         for (Adjacency adj : copyNewAdjs) {
1505                             if (copyOldAdjs.contains(adj)) {
1506                                 copyOldAdjs.remove(adj);
1507                             } else {
1508                                 // add new adjacency - right now only extra route will hit this path
1509                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1510                                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1511                                              dpnId, writeOperTxn, writeConfigTxn);
1512                                 }
1513                                 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to vpn "
1514                                                 + "interface {} on vpn {} dpnId {}",
1515                                         adj.getIpAddress(), adj.getNextHopIpList(),
1516                                         adj.getLabel(), adj.getSubnetId(), update.getName(),
1517                                         newVpnName, dpnId);
1518                             }
1519                         }
1520                         for (Adjacency adj : copyOldAdjs) {
1521                             if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1522                                 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1523                                        writeOperTxn, writeConfigTxn);
1524                             }
1525                             LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1526                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1527                                 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1528                         }
1529                     }
1530                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
1531                     try {
1532                         operFuture.get();
1533                     } catch (ExecutionException e) {
1534                         LOG.error("Exception encountered while submitting operational future for update"
1535                                 + " VpnInterface {} on vpn {}: {}", vpnInterfaceName, newVpnName, e);
1536                         return null;
1537                     }
1538                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1539                     futures.add(writeConfigTxn.submit());
1540                     LOG.info("update: vpn interface updated for interface {} oldVpn(s) {} newVpn {}"
1541                         + "processed successfully", update.getName(),
1542                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), newVpnName);
1543                     return futures;
1544                 });
1545             } else {
1546                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1547                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1548             }
1549         }
1550     }
1551
1552     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1553                                                  VpnInterface original, VpnInterface update) {
1554         boolean isSwap = Boolean.FALSE;
1555         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1556         final String interfaceName = key.getName();
1557         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1558             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1559         List<String> oldVpnListCopy = new ArrayList<>();
1560         oldVpnListCopy.addAll(oldVpnList);
1561         List<String> newVpnList = update.getVpnInstanceNames().stream()
1562             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1563         oldVpnList.removeAll(newVpnList);
1564         for (String oldVpnName: oldVpnList) {
1565             isSwap = Boolean.TRUE;
1566             LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {} running"
1567                      + " config-driven swap removal", interfaceName, oldVpnName);
1568             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1569             LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1570                      + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1571         }
1572         //Wait for previous interface bindings to be removed
1573         try {
1574             Thread.sleep(2000);
1575         } catch (InterruptedException e) {
1576             //Ignore
1577         }
1578         newVpnList.removeAll(oldVpnListCopy);
1579         for (String newVpnName: newVpnList) {
1580             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1581             isSwap = Boolean.TRUE;
1582             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1583                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1584                          + "running config-driven swap addition", interfaceName, newVpnName);
1585                 final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1586                 final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1587                                                 ? origAdjs.getAdjacency() : new ArrayList<>();
1588                 final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1589                 final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1590                                                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1591
1592                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1593                 LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1594                          + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1595                          interfaceName, oldVpnListCopy, newVpnName);
1596             }
1597         }
1598         return isSwap;
1599     }
1600
1601     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1602         Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1603         synchronized (label.toString().intern()) {
1604             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1605                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1606             Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1607             if (opResult.isPresent()) {
1608                 LabelRouteInfo labelRouteInfo =
1609                     new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1610                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1611             }
1612         }
1613         LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList);
1614     }
1615
1616     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1617         SubnetRoute route, WriteTransaction writeConfigTxn) {
1618
1619         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1620         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, null /* parentVpnRd */)
1621                 .addAugmentation(SubnetRoute.class, route).build();
1622         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1623         InstanceIdentifierBuilder<VrfTables> idBuilder =
1624             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1625         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1626         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1627         if (writeConfigTxn != null) {
1628             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1629         } else {
1630             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1631         }
1632         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1633                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1634     }
1635
1636     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1637                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1638                                            WriteTransaction writeConfigTxn) {
1639
1640         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1641                                                 LogicalDatastoreType.OPERATIONAL, identifier);
1642
1643         if (optVpnInterface.isPresent()) {
1644             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1645             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1646             String vpnName = currVpnIntf.getVpnInstanceName();
1647             VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1648             InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1649             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker,
1650                                             LogicalDatastoreType.OPERATIONAL, adjPath);
1651             boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1652             VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1653             long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1654             VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1655             List<Adjacency> adjacencies;
1656             if (optAdjacencies.isPresent()) {
1657                 adjacencies = optAdjacencies.get().getAdjacency();
1658             } else {
1659                 // This code will be hit in case of first PNF adjacency
1660                 adjacencies = new ArrayList<>();
1661             }
1662             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1663             L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1664                     .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1665             Adjacency operationalAdjacency = null;
1666             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) {
1667                 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1668                         : RouteOrigin.STATIC;
1669                 String nh = adj.getNextHopIpList().get(0);
1670                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1671                 synchronized (vpnPrefixKey.intern()) {
1672                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1673                             dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1674                     if (rdToAllocate.isPresent()) {
1675                         input.setRd(rdToAllocate.get());
1676                         operationalAdjacency = populator.createOperationalAdjacency(input);
1677                         int label = operationalAdjacency.getLabel().intValue();
1678                         vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1679                                 currVpnIntf.getVpnInstanceName(), l3vni, origin,
1680                                 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1681                         LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1682                                 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1683                                 vpnName, label, currVpnIntf.getName(), dpnId);
1684                     } else {
1685                         LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName,
1686                                 prefix);
1687                         return;
1688                     }
1689                     // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1690                     // Keeping the MPLS check for now.
1691                     if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1692                         final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1693                         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1694                                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1695                         vpnsToImportRoute.forEach(vpn -> {
1696                             if (vpn.getVrfId() != null) {
1697                                 VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1698                                         dataBroker, vpn.getVpnId(), vpnId, prefix,
1699                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1700                                         .ifPresent(
1701                                             rds -> vpnManager.addExtraRoute(
1702                                                     VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1703                                                     adj.getIpAddress(), nh, rds,
1704                                                     currVpnIntf.getVpnInstanceName(),
1705                                                     l3vni, RouteOrigin.SELF_IMPORTED,
1706                                                     currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn));
1707                             }
1708                         });
1709                     }
1710                 }
1711             } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1712                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1713                         currVpnIntf.getName(), vpnName);
1714
1715                 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1716
1717                 writeOperTxn.merge(
1718                         LogicalDatastoreType.OPERATIONAL,
1719                         VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1720                                 adj.getSubnetId().getValue()), prefix),
1721                         VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1722                                 adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1723
1724                 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1725                         adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, 0 /*l3vni*/,
1726                       null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1727
1728                 input.setRd(adj.getVrfId());
1729             }
1730             if (operationalAdjacency == null) {
1731                 operationalAdjacency = populator.createOperationalAdjacency(input);
1732             }
1733             adjacencies.add(operationalAdjacency);
1734             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1735             VpnInterfaceOpDataEntry newVpnIntf =
1736                 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1737                     aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1738                         currVpnIntf.getGatewayMacAddress());
1739
1740             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1741         }
1742     }
1743
1744     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1745         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1746         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1747     }
1748
1749     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1750             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1751         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1752                                           LogicalDatastoreType.OPERATIONAL, identifier);
1753
1754         if (optVpnInterface.isPresent()) {
1755             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1756
1757             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1758             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1759             if (optAdjacencies.isPresent()) {
1760                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1761
1762                 if (!adjacencies.isEmpty()) {
1763                     LOG.trace("delAdjFromVpnInterface: Adjacencies are " + adjacencies);
1764                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1765                     while (adjIt.hasNext()) {
1766                         Adjacency adjElem = adjIt.next();
1767                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1768                             String rd = adjElem.getVrfId();
1769                             adjIt.remove();
1770
1771                             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1772                             VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1773                                     .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1774                                     currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove(),
1775                                             currVpnIntf.getLportTag(), currVpnIntf.getGatewayMacAddress());
1776
1777                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1778                             if (adj.getNextHopIpList() != null) {
1779                                 for (String nh : adj.getNextHopIpList()) {
1780                                     deleteExtraRouteFromCurrentAndImportingVpns(
1781                                         currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1782                                         currVpnIntf.getName(), writeConfigTxn);
1783                                 }
1784                             } else if (adj.isPhysNetworkFunc()) {
1785                                 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet [}",
1786                                         adj.getIpAddress(), adj.getSubnetId());
1787                                 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1788                                         writeConfigTxn);
1789                             }
1790                             break;
1791                         }
1792
1793                     }
1794                 }
1795                 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1796                         dpnId, adj.getVrfId());
1797             } else {
1798                 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1799                         + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1800             }
1801         }
1802     }
1803
1804     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1805         String rd, String intfName, WriteTransaction writeConfigTxn) {
1806         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1807         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1808         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1809             String vpnRd = vpn.getVrfId();
1810             if (vpnRd != null) {
1811                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1812             }
1813         }
1814     }
1815
1816     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1817         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1818             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1819             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1820     }
1821
1822     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1823         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1824             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1825     }
1826
1827     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1828         WriteTransaction writeOperTxn) {
1829         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1830         if (dpId.equals(BigInteger.ZERO)) {
1831             LOG.error("addToNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1832                     + " association model", vpnInterfaceName, routerName);
1833
1834             return;
1835         }
1836         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1837
1838         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1839             .OPERATIONAL, routerDpnListIdentifier);
1840         RouterInterfaces routerInterface =
1841             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1842                 vpnInterfaceName).build();
1843         if (optionalRouterDpnList.isPresent()) {
1844             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1845                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1846         } else {
1847             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1848             builder.setRouterId(routerName);
1849             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1850             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
1851             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1852                 getRouterId(routerName),
1853                 builder.build(), true);
1854         }
1855     }
1856
1857     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1858         WriteTransaction writeOperTxn) {
1859         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1860         if (dpId.equals(BigInteger.ZERO)) {
1861             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1862                     + " dissociation model", vpnInterfaceName, routerName);
1863
1864             return;
1865         }
1866         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1867         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1868             .OPERATIONAL, routerDpnListIdentifier);
1869         if (optionalRouterDpnList.isPresent()) {
1870             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1871             RouterInterfaces routerInterface =
1872                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1873                     vpnInterfaceName).build();
1874
1875             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1876                 if (routerInterfaces.isEmpty()) {
1877                     if (writeOperTxn != null) {
1878                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1879                     } else {
1880                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1881                     }
1882                 } else {
1883                     if (writeOperTxn != null) {
1884                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1885                             RouterInterfaces.class,
1886                             new RouterInterfacesKey(vpnInterfaceName)));
1887                     } else {
1888                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1889                             routerDpnListIdentifier.child(
1890                                 RouterInterfaces.class,
1891                                 new RouterInterfacesKey(vpnInterfaceName)));
1892                     }
1893                 }
1894             }
1895         }
1896     }
1897
1898     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1899         WriteTransaction writeOperTxn) {
1900         if (dpId.equals(BigInteger.ZERO)) {
1901             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1902                     + " dissociation model", vpnInterfaceName, routerName);
1903
1904             return;
1905         }
1906         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1907         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1908             .OPERATIONAL, routerDpnListIdentifier);
1909         if (optionalRouterDpnList.isPresent()) {
1910             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1911             RouterInterfaces routerInterface =
1912                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1913                     vpnInterfaceName).build();
1914             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1915                 if (routerInterfaces.isEmpty()) {
1916                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1917                 } else {
1918                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1919                         RouterInterfaces.class,
1920                         new RouterInterfacesKey(vpnInterfaceName)));
1921                 }
1922             }
1923         }
1924     }
1925
1926     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1927                                                     WriteTransaction writeConfigTxn, String vpnName) {
1928         if (vpnInterface == null) {
1929             return;
1930         }
1931         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1932         if (adjs == null) {
1933             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1934                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1935             return;
1936         }
1937         for (Adjacency adj : adjs) {
1938             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1939                 String primaryInterfaceIp = adj.getIpAddress();
1940                 String macAddress = adj.getMacAddress();
1941                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1942
1943                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1944                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1945
1946                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1947                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1948                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1949                         routerInt, label, writeConfigTxn);
1950                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1951                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1952                         macAddress);
1953                 return;
1954             }
1955         }
1956         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1957                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1958                 primaryRd, vpnName);
1959     }
1960
1961     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1962                                                     WriteTransaction writeConfigTxn, String vpnName) {
1963         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
1964         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1965         if (adjs != null) {
1966             List<Adjacency> adjsList = adjs.getAdjacency();
1967             for (Adjacency adj : adjsList) {
1968                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1969                     String primaryInterfaceIp = adj.getIpAddress();
1970                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1971                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1972                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1973                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1974                     return;
1975                 }
1976             }
1977         } else {
1978             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1979                     vpnInterface.getName(), rd);
1980         }
1981     }
1982
1983     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1984         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1985             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1986                   vpnName, intefaceData.vpnInterface.getName());
1987             return;
1988         }
1989         final VpnInterfaceKey key = intefaceData.identifier
1990                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1991         final String interfaceName = key.getName();
1992         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1993                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1994         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1995                   intefaceData.identifier, vpnName);
1996         return;
1997     }
1998
1999     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2000                                               VpnInterface vpnInterface, String vpnName) {
2001         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2002                .get(vpnName);
2003         if (vpnInterfaces == null) {
2004             vpnInterfaces = new ConcurrentLinkedQueue<>();
2005         }
2006         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2007         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2008         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2009                 vpnInterface.getName(), vpnName);
2010     }
2011
2012     public boolean isVpnInstanceReady(String vpnInstanceName) {
2013         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2014         if (vpnRd == null) {
2015             return false;
2016         }
2017         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2018
2019         return vpnInstanceOpDataEntry != null;
2020     }
2021
2022     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2023         synchronized (vpnInstanceName.intern()) {
2024             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2025                     unprocessedVpnInterfaces.get(vpnInstanceName);
2026             if (vpnInterfaces != null) {
2027                 while (!vpnInterfaces.isEmpty()) {
2028                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2029                     if (hasVpnInstanceCreatedSuccessfully) {
2030                         processSavedInterface(savedInterface, vpnInstanceName);
2031                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2032                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2033                     } else {
2034                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2035                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2036                     }
2037                 }
2038             } else {
2039                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2040             }
2041         }
2042     }
2043
2044     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2045             VpnInterface vpnInterface) {
2046         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2047             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2048                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2049             if (vpnInterfaces != null) {
2050                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2051                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2052                             + "unprocessed list", vpnInterface.getName(),
2053                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2054                 }
2055             } else {
2056                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2057                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2058             }
2059         }
2060     }
2061
2062     public void vpnInstanceIsReady(String vpnInstanceName) {
2063         processSavedInterfaces(vpnInstanceName, true);
2064     }
2065
2066     public void vpnInstanceFailed(String vpnInstanceName) {
2067         processSavedInterfaces(vpnInstanceName, false);
2068     }
2069
2070     private static class UnprocessedVpnInterfaceData {
2071         InstanceIdentifier<VpnInterface> identifier;
2072         VpnInterface vpnInterface;
2073
2074         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2075             this.identifier = identifier;
2076             this.vpnInterface = vpnInterface;
2077         }
2078
2079         @Override
2080         public int hashCode() {
2081             final int prime = 31;
2082             int result = 1;
2083             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2084             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2085             return result;
2086         }
2087
2088         @Override
2089         public boolean equals(Object obj) {
2090             if (this == obj) {
2091                 return true;
2092             }
2093             if (obj == null) {
2094                 return false;
2095             }
2096             if (getClass() != obj.getClass()) {
2097                 return false;
2098             }
2099             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2100             if (identifier == null) {
2101                 if (other.identifier != null) {
2102                     return false;
2103                 }
2104             } else if (!identifier.equals(other.identifier)) {
2105                 return false;
2106             }
2107             if (vpnInterface == null) {
2108                 if (other.vpnInterface != null) {
2109                     return false;
2110                 }
2111             } else if (!vpnInterface.equals(other.vpnInterface)) {
2112                 return false;
2113             }
2114             return true;
2115         }
2116     }
2117
2118     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2119         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2120         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2121         if (vpnInstanceOpData == null) {
2122             return;
2123         }
2124         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2125         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2126             return;
2127         }
2128         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2129         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2130             InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2131                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2132             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = VpnUtil.read(dataBroker,
2133                     LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2134             if (!vpnInterfaceOptional.isPresent()) {
2135                 return;
2136             }
2137             List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2138                     vpnInterface.getInterfaceName());
2139             if (configVpnAdjacencies == null) {
2140                 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2141                 return;
2142             }
2143             List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2144                     .getAugmentation(AdjacenciesOp.class).getAdjacency();
2145             // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2146             // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2147             // in operational DS. These unprocessed adjacencies will be handled below.
2148             // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2149             // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2150             configVpnAdjacencies.stream()
2151                     .filter(adjacency -> operationVpnAdjacencies.stream()
2152                             .noneMatch(operationalAdjacency ->
2153                                     operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2154                     .forEach(adjacency -> {
2155                         LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2156                         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2157                             () -> {
2158                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
2159                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
2160                                 if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2161                                     addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd, adjacency,
2162                                             vpnInterfaceOptional.get().getDpnId(), writeConfigTxn, writeOperTxn);
2163                                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
2164                                     try {
2165                                         operFuture.get();
2166                                     } catch (ExecutionException | InterruptedException e) {
2167                                         LOG.error("Exception encountered while submitting operational"
2168                                                 + " future for vpnInterface {}", vpnInterface, e);
2169                                     }
2170                                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2171                                     futures.add(writeConfigTxn.submit());
2172                                     return futures;
2173                                 }  else {
2174                                     return Collections.emptyList();
2175                                 }
2176                             });
2177                     });
2178         }));
2179     }
2180
2181     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2182         private final String interfaceName;
2183         private final boolean add;
2184         private final String txnDestination;
2185
2186         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2187             this.interfaceName = interfaceName;
2188             this.add = add;
2189             this.txnDestination = transactionDest;
2190         }
2191
2192         @Override
2193         public void onSuccess(Void voidObj) {
2194             if (add) {
2195                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2196                         interfaceName, txnDestination);
2197             } else {
2198                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2199             }
2200         }
2201
2202         @Override
2203         public void onFailure(Throwable throwable) {
2204             if (add) {
2205                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}"
2206                         + " with exception: {}", interfaceName, txnDestination, throwable);
2207             } else {
2208                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed with exception: {}", interfaceName,
2209                         throwable);
2210                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
2211             }
2212         }
2213     }
2214 }