vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.concurrent.ConcurrentHashMap;
  25 import java.util.concurrent.ConcurrentLinkedQueue;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  42 import org.opendaylight.genius.mdsalutil.NWUtil;
  43 import org.opendaylight.genius.mdsalutil.NwConstants;
  44 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.infrautils.caches.CacheProvider;
  47 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  51 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  52 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  53 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  54 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  55 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  56 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  57 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  58 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  59 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 109 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 110 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 111 import org.slf4j.Logger;
 112 import org.slf4j.LoggerFactory;
 113
 114 @Singleton
 115 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 116
 117     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 118     private static final short DJC_MAX_RETRIES = 3;
 119
 120     private final DataBroker dataBroker;
 121     private final ManagedNewTransactionRunner txRunner;
 122     private final IBgpManager bgpManager;
 123     private final IFibManager fibManager;
 124     private final IMdsalApiManager mdsalManager;
 125     private final IdManagerService idManager;
 126     private final OdlInterfaceRpcService ifaceMgrRpcService;
 127     private final VpnFootprintService vpnFootprintService;
 128     private final IInterfaceManager interfaceManager;
 129     private final IVpnManager vpnManager;
 130     private final ArpResponderHandler arpResponderHandler;
 131     private final JobCoordinator jobCoordinator;
 132
 133     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 134
 135     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 136             new ConcurrentHashMap<>();
 137
 138     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 139
 140     @Inject
 141     public VpnInterfaceManager(final DataBroker dataBroker,
 142                                final IBgpManager bgpManager,
 143                                final IdManagerService idManager,
 144                                final IMdsalApiManager mdsalManager,
 145                                final IFibManager fibManager,
 146                                final OdlInterfaceRpcService ifaceMgrRpcService,
 147                                final VpnFootprintService vpnFootprintService,
 148                                final IInterfaceManager interfaceManager,
 149                                final IVpnManager vpnManager,
 150                                final ArpResponderHandler arpResponderHandler,
 151                                final JobCoordinator jobCoordinator,
 152                                final CacheProvider cacheProvider) {
 153         super(VpnInterface.class, VpnInterfaceManager.class);
 154
 155         this.dataBroker = dataBroker;
 156         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 157         this.bgpManager = bgpManager;
 158         this.idManager = idManager;
 159         this.mdsalManager = mdsalManager;
 160         this.fibManager = fibManager;
 161         this.ifaceMgrRpcService = ifaceMgrRpcService;
 162         this.vpnFootprintService = vpnFootprintService;
 163         this.interfaceManager = interfaceManager;
 164         this.vpnManager = vpnManager;
 165         this.arpResponderHandler = arpResponderHandler;
 166         this.jobCoordinator = jobCoordinator;
 167
 168         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 169                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 170                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 171     }
 172
 173     public Runnable isNotifyTaskQueued(String intfName) {
 174         return vpnIntfMap.remove(intfName);
 175     }
 176
 177     @PostConstruct
 178     public void start() {
 179         LOG.info("{} start", getClass().getSimpleName());
 180         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 181     }
 182
 183     @Override
 184     @PreDestroy
 185     public void close() {
 186         super.close();
 187         vpnInstanceOpDataEntryCache.close();
 188     }
 189
 190     @Override
 191     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 192         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 193     }
 194
 195     @Override
 196     protected VpnInterfaceManager getDataTreeChangeListener() {
 197         return VpnInterfaceManager.this;
 198     }
 199
 200     @Override
 201     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 202         LOG.info("add: intfName {} onto vpnName {}",
 203                  vpnInterface.getName(),
 204                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 205         addVpnInterface(identifier, vpnInterface, null, null);
 206     }
 207
 208     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 209                           final VpnInterface vpnInterface, String vpnName) {
 210         synchronized (vpnName.intern()) {
 211             if (isVpnInstanceReady(vpnName)) {
 212                 return true;
 213             }
 214             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 215             return false;
 216         }
 217     }
 218
 219     // TODO Clean up the exception handling
 220     @SuppressWarnings("checkstyle:IllegalCatch")
 221     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 222                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 223         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 224             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 225             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 226         }
 227     }
 228
 229     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 230                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 231         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 232         final String interfaceName = key.getName();
 233
 234         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 235             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 236                   vpnName, vpnInterface.getName());
 237             return;
 238         }
 239         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 240              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 241         List<Adjacency> copyOldAdjs = null;
 242         if (oldAdjs != null) {
 243             copyOldAdjs = new ArrayList<>();
 244             copyOldAdjs.addAll(oldAdjs);
 245         }
 246         List<Adjacency> copyNewAdjs = null;
 247         if (newAdjs != null) {
 248             copyNewAdjs = new ArrayList<>();
 249             copyNewAdjs.addAll(newAdjs);
 250         }
 251         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 252     }
 253
 254     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 255                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 256                     final List<Adjacency> newAdjs,
 257                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 258         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 259         final String interfaceName = key.getName();
 260         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 261         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 262             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 263             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 264             if (interfaceState != null) {
 265                 try {
 266                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 267                     final int ifIndex = interfaceState.getIfIndex();
 268                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 269                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 270                         WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 271                         WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 272                         LOG.info("addVpnInterface: VPN Interface add event - intfName {} vpnName {} on dpn {}" ,
 273                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 274                         processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, writeConfigTxn,
 275                                      writeOperTxn, writeInvTxn, interfaceState, vpnName);
 276                         if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 277                             LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 278                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 279                             if (newAdjs != null) {
 280                                 for (Adjacency adj : newAdjs) {
 281                                     if (oldAdjs.contains(adj)) {
 282                                         oldAdjs.remove(adj);
 283                                     } else {
 284                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 285                                                                   dataBroker, adj)) {
 286                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd,
 287                                                   adj, dpnId, writeOperTxn, writeConfigTxn);
 288                                         }
 289                                     }
 290                                 }
 291                             }
 292                             for (Adjacency adj : oldAdjs) {
 293                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 294                                                           dataBroker, adj)) {
 295                                     delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 296                                         writeOperTxn, writeConfigTxn);
 297                                 }
 298                             }
 299                         }
 300                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
 301                         try {
 302                             operFuture.get();
 303                         } catch (ExecutionException e) {
 304                             LOG.error("addVpnInterface: Exception encountered while submitting operational future for"
 305                                     + " addVpnInterface {} on vpn {}", vpnInterface.getName(), vpnName, e);
 306                             return null;
 307                         }
 308                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 309                         ListenableFuture<Void> configFuture = writeConfigTxn.submit();
 310                         futures.add(configFuture);
 311                         Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"));
 312                         futures.add(writeInvTxn.submit());
 313                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 314                                 + " processed successfully", interfaceName, vpnName, dpnId);
 315                         return futures;
 316                     });
 317                 } catch (NumberFormatException | IllegalStateException e) {
 318                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 319                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 320                                     vpnName, e);
 321                     return;
 322                 }
 323             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 324                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 325                     () -> {
 326                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 327                         createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 328                                        writeConfigTxn, vpnName);
 329                         LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 330                                  vpnName, vpnInterface.getDpnId());
 331                         ListenableFuture<Void> futures = writeConfigTxn.submit();
 332                         String errorText = "addVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
 333                              + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
 334                         ListenableFutures.addErrorLogging(futures, LOG, errorText);
 335                         return Collections.singletonList(futures);
 336                     });
 337             } else {
 338                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 339                         + " is not available", interfaceName, vpnName);
 340             }
 341         } else {
 342             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 343                       + " as vpn is pending delete", interfaceName, vpnName,
 344                     vpnInterface.getDpnId());
 345         }
 346     }
 347
 348     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 349     // see comments below.
 350     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 351     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 352             final int lportTag, boolean isInterfaceUp,
 353             WriteTransaction writeConfigTxn,
 354             WriteTransaction writeOperTxn,
 355             WriteTransaction writeInvTxn,
 356             Interface interfaceState,
 357             final String vpnName) {
 358         final String interfaceName = vpnInterface.getName();
 359         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 360                                                          interfaceName, vpnName);
 361         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 362         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 363         if (!isInterfaceUp) {
 364             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 365                      interfaceName, dpId, vpnName);
 366             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 367             if (vpnId == VpnConstants.INVALID_ID) {
 368                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 369                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 370                         dpId);
 371                 return;
 372             }
 373
 374             boolean waitForVpnInterfaceOpRemoval = false;
 375             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 376                 String opVpnName = opVpnInterface.getVpnInstanceName();
 377                 String primaryInterfaceIp = null;
 378                 if (opVpnName.equals(vpnName)) {
 379                     // Please check if the primary VRF Entry does not exist for VPNInterface
 380                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 381                     // back to back
 382                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 383                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 384                     if (adjs == null) {
 385                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 386                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 387                                 vpnName);
 388                         return;
 389                     }
 390                     for (Adjacency adj : adjs) {
 391                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 392                             primaryInterfaceIp = adj.getIpAddress();
 393                             break;
 394                         }
 395                     }
 396                     if (primaryInterfaceIp == null) {
 397                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 398                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 399                                 dpId, vpnName);
 400                         return;
 401                     }
 402                     // Get the rd of the vpn instance
 403                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 404                     if (vrf != null) {
 405                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 406                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 407                         return;
 408                     }
 409                     waitForVpnInterfaceOpRemoval = true;
 410                 } else {
 411                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 412                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 413                 }
 414             }
 415             if (!waitForVpnInterfaceOpRemoval) {
 416                 // Add the VPNInterface and quit
 417                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 418                         null/*ipAddressSourceValuePair*/,
 419                         true /* add */);
 420                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 421                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 422                 if (!isBgpVpnInternetVpn) {
 423                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 424                             jobCoordinator);
 425                 }
 426                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 427                         dpId, vpnName);
 428                 if (interfaceManager.isExternalInterface(interfaceName)) {
 429                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 430                         NwConstants.ADD_FLOW);
 431                 }
 432                 return;
 433             }
 434
 435             // FIB didn't get a chance yet to clean up this VPNInterface
 436             // Let us give it a chance here !
 437             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 438                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 439             try {
 440                 Runnable notifyTask = new VpnNotifyTask();
 441                 synchronized (notifyTask) {
 442                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 443                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 444                     // published here while holding the lock on it so this path will hit the wait before notify can be
 445                     // invoked.
 446                     vpnIntfMap.put(interfaceName, notifyTask);
 447                     try {
 448                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 449                     } catch (InterruptedException e) {
 450                         // Ignored
 451                     }
 452                 }
 453             } finally {
 454                 vpnIntfMap.remove(interfaceName);
 455             }
 456
 457             if (opVpnInterface != null) {
 458                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 459                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 460                         dpId, vpnName);
 461                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
 462                 return;
 463             }
 464             // VPNInterface got removed, proceed with Add
 465             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 466                     interfaceName, dpId, vpnName);
 467             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 468                     null/*ipAddressSourceValuePair*/,
 469                     true /* add */);
 470             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 471                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 472             if (!isBgpVpnInternetVpn) {
 473                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 474             }
 475             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 476                     + " FIB to clean up", interfaceName, dpId, vpnName);
 477             if (interfaceManager.isExternalInterface(interfaceName)) {
 478                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 479                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 480             }
 481         } else {
 482             // Interface is retained in the DPN, but its Link Up.
 483             // Advertise prefixes again for this interface to BGP
 484             InstanceIdentifier<VpnInterface> identifier =
 485                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 486             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 487                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 488             advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 489             // Perform similar operation as interface add event for extraroutes.
 490             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 491             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 492             if (!optAdjacencies.isPresent()) {
 493                 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 494                 return;
 495             }
 496             List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 497             for (Adjacency adjacency : adjacencies) {
 498                 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 499                     continue;
 500                 }
 501                 // if BGPVPN Internet, filter only IPv6 Adjacencies
 502                 if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 503                                 dataBroker, adjacency)) {
 504                     continue;
 505                 }
 506                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 507                           dpId, writeOperTxn, writeConfigTxn);
 508             }
 509         }
 510     }
 511
 512     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 513             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 514         Uuid extNetworkId;
 515         try {
 516             // vpn instance of ext-net interface is the network-id
 517             extNetworkId = new Uuid(vpnName);
 518         } catch (IllegalArgumentException e) {
 519             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 520                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 521             return;
 522         }
 523
 524         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 525         if (routerIds == null || routerIds.isEmpty()) {
 526             LOG.info("processExternalVpnInterface: No router is associated with {}."
 527                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 528                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 529             return;
 530         }
 531
 532         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 533                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 534         for (Uuid routerId : routerIds) {
 535             String routerName = routerId.getValue();
 536             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 537             if (Objects.equals(primarySwitch, dpId)) {
 538                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 539                 if (router != null) {
 540                     if (addOrRemove == NwConstants.ADD_FLOW) {
 541                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 542                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 543                                 dpId, interfaceName, lportTag, writeInvTxn);
 544                     } else {
 545                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 546                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 547                                 dpId, interfaceName, lportTag);
 548                     }
 549                 } else {
 550                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 551                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 552                             interfaceName, dpId, vpnName);
 553                 }
 554             }
 555         }
 556     }
 557
 558     // TODO Clean up the exception handling
 559     @SuppressWarnings("checkstyle:IllegalCatch")
 560     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 561                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 562                                                   String vpnName, String interfaceName) {
 563         if (rd == null) {
 564             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 565                   interfaceName, dpnId, vpnName);
 566             return;
 567         } else {
 568             if (rd.equals(vpnName)) {
 569                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 570                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 571                 return;
 572             }
 573         }
 574         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 575                 interfaceName, dpnId, vpnName, rd);
 576
 577         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 578         if (nextHopIp == null) {
 579             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 580                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 581                     rd, vpnName);
 582             return;
 583         }
 584
 585         //Read NextHops
 586         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 587         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 588         if (adjacencies.isPresent()) {
 589             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 590
 591             if (!nextHops.isEmpty()) {
 592                 LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 593                         + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 594                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 595                 long l3vni = vpnInstanceOpData.getL3vni();
 596                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 597                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 598                 for (Adjacency nextHop : nextHops) {
 599                     if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 600                         continue;
 601                     }
 602                     String gatewayMac = null;
 603                     long label = 0;
 604                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 605                         final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 606                             vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 607                         gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName).get();
 608                     } else {
 609                         label = nextHop.getLabel();
 610                     }
 611                     try {
 612                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 613                                 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 614                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 615                                 encapType, (int)label, l3vni, 0 /*l2vni*/,
 616                                 gatewayMac);
 617                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 618                                 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 619                                 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 620                     } catch (Exception e) {
 621                         LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}"
 622                                 + " for interface {} on dpn {}", nextHop.getIpAddress(),
 623                                   vpnName, rd, interfaceName, dpnId, e);
 624                     }
 625                 }
 626             }
 627         }
 628     }
 629
 630     // TODO Clean up the exception handling
 631     @SuppressWarnings("checkstyle:IllegalCatch")
 632     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 633                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 634         //Read NextHops
 635         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 636         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 637
 638         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 639         if (rd == null) {
 640             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 641                 interfaceName, vpnName);
 642             return;
 643         } else {
 644             if (rd.equals(vpnName)) {
 645                 LOG.info(
 646                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 647                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 648                 return;
 649             }
 650         }
 651         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 652                vpnName, rd);
 653         if (adjacencies.isPresent()) {
 654             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 655
 656             if (!nextHops.isEmpty()) {
 657                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 658                         nextHops, interfaceName, vpnName, rd);
 659                 for (Adjacency nextHop : nextHops) {
 660                     try {
 661                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 662                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 663                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 664                                     interfaceName, vpnName);
 665                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 666                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 667                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 668                                     interfaceName, vpnName);
 669                         } else {
 670                             // Perform similar operation as interface delete event for extraroutes.
 671                             String allocatedRd = nextHop.getVrfId();
 672                             for (String nh : nextHop.getNextHopIpList()) {
 673                                 deleteExtraRouteFromCurrentAndImportingVpns(
 674                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 675                             }
 676                         }
 677                     } catch (Exception e) {
 678                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 679                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 680                     }
 681                 }
 682             }
 683         }
 684     }
 685
 686     @SuppressWarnings("checkstyle:IllegalCatch")
 687     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 688                                                   String primaryRd, String interfaceName, final long vpnId,
 689                                                   WriteTransaction writeConfigTxn,
 690                                                   WriteTransaction writeOperTxn,
 691                                                   final WriteTransaction writeInvTxn,
 692                                                   Interface interfaceState) {
 693         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 694         // Read NextHops
 695         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 696         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 697         if (!adjacencies.isPresent()) {
 698             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 699                     null/*gwMac*/, writeOperTxn);
 700             return;
 701         }
 702
 703         // Get the rd of the vpn instance
 704         String nextHopIp = null;
 705         try {
 706             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 707         } catch (Exception e) {
 708             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 709                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 710         }
 711         List<String> nhList = new ArrayList<>();
 712         if (nextHopIp != null) {
 713             nhList.add(nextHopIp);
 714             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 715                     interfaceName, dpnId, vpnName, nhList);
 716         }
 717         Optional<String> gwMac = Optional.absent();
 718         String vpnInterfaceSubnetGwMacAddress = null;
 719         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 720         Long l3vni = vpnInstanceOpData.getL3vni();
 721         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 722         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 723         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 724         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 725         List<Adjacency> value = new ArrayList<>();
 726         for (Adjacency nextHop : nextHops) {
 727             String rd = primaryRd;
 728             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 729             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 730                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 731                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 732                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 733                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 734                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 735                 continue;
 736             }
 737             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 738                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 739                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 740                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 741                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 742                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 743                 writeOperTxn.merge(
 744                     LogicalDatastoreType.OPERATIONAL,
 745                     VpnUtil.getPrefixToInterfaceIdentifier(
 746                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 747                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 748                             prefixCue), true);
 749                 final Uuid subnetId = nextHop.getSubnetId();
 750
 751                 String gatewayIp = nextHop.getSubnetGatewayIp();
 752                 if (gatewayIp == null) {
 753                     Optional<String> gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 754                     if (gatewayIpOptional.isPresent()) {
 755                         gatewayIp = gatewayIpOptional.get();
 756                     }
 757                 }
 758
 759                 if (gatewayIp != null) {
 760                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 761                     if (gwMac.isPresent()) {
 762                         // A valid mac-address is available for this subnet-gateway-ip
 763                         // Use this for programming ARP_RESPONDER table here.  And save this
 764                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 765                         // to populate L3_GW_MAC_TABLE there.
 766                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 767                                 gatewayIp, gwMac.get());
 768                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 769                     } else {
 770                         // A valid mac-address is not available for this subnet-gateway-ip
 771                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 772                         // Save this connected-mac-address as gateway-mac-address for the
 773                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 774                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 775                         if (gwMac.isPresent()) {
 776                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 777                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 778                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 779                                     gatewayIp, gwMac.get());
 780                         } else {
 781                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 782                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 783                                 + "gwIp {}",
 784                                 subnetId, interfaceName, vpnName, gatewayIp);
 785                         }
 786                     }
 787                 } else {
 788                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 789                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 790                         subnetId, interfaceName, vpnName);
 791                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 792                 }
 793                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 794                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 795             } else {
 796                 //Extra route adjacency
 797                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 798                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 799                 synchronized (vpnPrefixKey.intern()) {
 800                     java.util.Optional<String> rdToAllocate = VpnUtil
 801                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 802                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 803                     if (rdToAllocate.isPresent()) {
 804                         rd = rdToAllocate.get();
 805                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 806                             rd, prefix);
 807                     } else {
 808                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 809                         continue;
 810                     }
 811                 }
 812                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 813                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 814                         interfaceName, dpnId);
 815             }
 816             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 817             // can be different from the gateway-mac-address within the VRFEntry as the
 818             // gateway-mac-address is a superset.
 819             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 820                     : RouteOrigin.STATIC;
 821             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 822                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 823                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 824             Adjacency operationalAdjacency = null;
 825             try {
 826                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 827             } catch (NullPointerException e) {
 828                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 829                     input, e.getMessage());
 830                 return;
 831             }
 832             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 833                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 834                         vpnName, l3vni, origin,
 835                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 836             }
 837             value.add(operationalAdjacency);
 838         }
 839
 840         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 841         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 842                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 843
 844         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 845                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 846                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 847
 848         for (Adjacency nextHop : aug.getAdjacency()) {
 849             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 850             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 851                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 852                         : RouteOrigin.STATIC;
 853                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 854                 registeredPopulator.populateFib(input, writeConfigTxn);
 855             }
 856         }
 857     }
 858
 859     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 860                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 861         VpnInterfaceOpDataEntry opInterface =
 862               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 863         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 864             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 865         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 866                 WriteTransaction.CREATE_MISSING_PARENTS);
 867         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 868                 interfaceName, dpnId, vpnName);
 869     }
 870
 871     // TODO Clean up the exception handling
 872     @SuppressWarnings("checkstyle:IllegalCatch")
 873     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 874                                            StateTunnelList stateTunnelList,
 875                                            WriteTransaction writeConfigTxn,
 876                                            WriteTransaction writeOperTxn) {
 877
 878         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 879         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 880         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 881         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 882         if (adjList.isEmpty()) {
 883             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 884                     vpnInterface, srcDpnId);
 885             return;
 886         }
 887         String prefix = null;
 888         long label = 0;
 889         List<Adjacency> value = new ArrayList<>();
 890         boolean isNextHopAddReqd = false;
 891         String vpnName = vpnInterface.getVpnInstanceName();
 892         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 893         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 894         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 895                 vpnInterface.getName(), vpnInterface.getDpnId(),
 896                 vpnInterface.getVpnInstanceName(), adjList);
 897         for (Adjacency adj : adjList) {
 898             String rd = adj.getVrfId();
 899             rd = rd != null ? rd : vpnName;
 900             prefix = adj.getIpAddress();
 901             label = adj.getLabel();
 902             List<String> nhList = Collections.singletonList(srcTepIp);
 903             List<String> nextHopList = adj.getNextHopIpList();
 904             // If TEP is added , update the nexthop of primary adjacency.
 905             // Secondary adj nexthop is already pointing to primary adj IP address.
 906             if (nextHopList != null && !nextHopList.isEmpty() && nextHopList.get(0).equalsIgnoreCase(srcTepIp)) {
 907                 /* everything right already */
 908             } else {
 909                 isNextHopAddReqd = true;
 910             }
 911
 912             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 913                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 914             } else {
 915                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 916                 if (!vrfEntryOptional.isPresent()) {
 917                     continue;
 918                 }
 919                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 920                 if (!nhList.contains(srcTepIp)) {
 921                     nhList.add(srcTepIp);
 922                     isNextHopAddReqd = true;
 923                 }
 924                 value.add(adj);
 925             }
 926
 927             if (isNextHopAddReqd) {
 928                 updateLabelMapper(label, nhList);
 929                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 930                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 931                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 932                 // Update the VRF entry with nextHop
 933                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 934                         label, true, writeConfigTxn);
 935
 936                 //Get the list of VPN's importing this route(prefix) .
 937                 // Then update the VRF entry with nhList
 938                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 939                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 940                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 941                     String vpnRd = vpn.getVrfId();
 942                     if (vpnRd != null) {
 943                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 944                                 srcTepIp, label, true, writeConfigTxn);
 945                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 946                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 947                             vpnInterface.getName(), srcDpnId, vpnName,
 948                             vpn.getVpnInstanceName(), vpnRd);
 949                     }
 950                 }
 951                 // Advertise the prefix to BGP only for external vpn
 952                 // since there is a nexthop change.
 953                 try {
 954                     if (!rd.equalsIgnoreCase(vpnName)) {
 955                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 956                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 957                                 null /*gatewayMacAddress*/);
 958                     }
 959                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 960                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 961                             srcDpnId, vpnName);
 962                 } catch (Exception ex) {
 963                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 964                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 965                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 966                 }
 967             }
 968         }
 969         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 970         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 971                 .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 972                 .addAugmentation(AdjacenciesOp.class, aug).build();
 973         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 974                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 975         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 976                 WriteTransaction.CREATE_MISSING_PARENTS);
 977         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 978                 vpnInterface.getName(), srcDpnId, vpnName);
 979
 980     }
 981
 982     // TODO Clean up the exception handling
 983     @SuppressWarnings("checkstyle:IllegalCatch")
 984     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
 985                                               StateTunnelList stateTunnelList,
 986                                               WriteTransaction writeConfigTxn,
 987                                               WriteTransaction writeOperTxn) {
 988
 989         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 990         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 991         String prefix = null;
 992         long label = 0;
 993         boolean isNextHopRemoveReqd = false;
 994         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 995         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 996         String vpnName = vpnInterface.getVpnInstanceName();
 997         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 998         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 999         if (adjList != null) {
1000             List<Adjacency> value = new ArrayList<>();
1001             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1002                     vpnInterface.getName(), vpnInterface.getDpnId(),
1003                     vpnInterface.getVpnInstanceName(), adjList);
1004             for (Adjacency adj : adjList) {
1005                 List<String> nhList = new ArrayList<>();
1006                 String rd = adj.getVrfId();
1007                 rd = rd != null ? rd : vpnName;
1008                 prefix = adj.getIpAddress();
1009                 List<String> nextHopList = adj.getNextHopIpList();
1010                 label = adj.getLabel();
1011                 if (nextHopList != null && !nextHopList.isEmpty()) {
1012                     isNextHopRemoveReqd = true;
1013                 }
1014                 // If TEP is deleted , remove the nexthop from primary adjacency.
1015                 // Secondary adj nexthop will continue to point to primary adj IP address.
1016                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1017                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1018                 } else {
1019                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1020                     if (!vrfEntryOptional.isPresent()) {
1021                         continue;
1022                     }
1023                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1024                     if (nhList.contains(srcTepIp)) {
1025                         nhList.remove(srcTepIp);
1026                         isNextHopRemoveReqd = true;
1027                     }
1028                     value.add(adj);
1029                 }
1030
1031                 if (isNextHopRemoveReqd) {
1032                     updateLabelMapper(label, nhList);
1033                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1034                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1035                             prefix, nhList, vpnName,
1036                             vpnId, rd, vpnInterface.getName());
1037                     // Update the VRF entry with removed nextHop
1038                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1039                             label, false, writeConfigTxn);
1040
1041                     //Get the list of VPN's importing this route(prefix) .
1042                     // Then update the VRF entry with nhList
1043                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1044                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1045                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1046                         String vpnRd = vpn.getVrfId();
1047                         if (vpnRd != null) {
1048                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1049                                     srcTepIp, label, false, writeConfigTxn);
1050                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1051                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1052                                     nhList, label, vpnInterface.getName(), srcDpnId,
1053                                     vpnName,
1054                                     vpn.getVpnInstanceName(), vpnRd);
1055                         }
1056                     }
1057
1058                     // Withdraw prefix from BGP only for external vpn.
1059                     try {
1060                         if (!rd.equalsIgnoreCase(vpnName)) {
1061                             bgpManager.withdrawPrefix(rd, prefix);
1062                         }
1063                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1064                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1065                                 vpnInterface.getName(), srcDpnId,
1066                                 vpnName);
1067                     } catch (Exception ex) {
1068                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1069                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1070                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1071                     }
1072                 }
1073             }
1074             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1075             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1076                     .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1077                     .addAugmentation(AdjacenciesOp.class, aug).build();
1078             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1079                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1080             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1081                     WriteTransaction.CREATE_MISSING_PARENTS);
1082             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1083                          vpnInterface.getName(), srcDpnId, vpnName);
1084         }
1085     }
1086
1087     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1088         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1089
1090         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1091         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1092         if (vpnInstanceOpDataEntry == null) {
1093             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1094                     + " to check for vpns exporting the routes", vpnName);
1095             return vpnsToExportRoute;
1096         }
1097
1098         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1099             if (input.getVpnInstanceName() == null) {
1100                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1101                         input.getVrfId());
1102                 return false;
1103             }
1104             return !input.getVpnInstanceName().equals(vpnName);
1105         };
1106
1107         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1108             Iterable<String> commonRTs =
1109                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1110                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1111             return Iterators.size(commonRTs.iterator()) > 0;
1112         };
1113
1114         vpnsToExportRoute =
1115                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1116                         Collectors.toList());
1117         return vpnsToExportRoute;
1118     }
1119
1120     // TODO Clean up the exception handling
1121     @SuppressWarnings("checkstyle:IllegalCatch")
1122     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1123         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1124         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1125             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1126             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1127             if (vrfEntries != null) {
1128                 for (VrfEntry vrfEntry : vrfEntries) {
1129                     try {
1130                         if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1131                                 RouteOrigin.value(vrfEntry.getOrigin()))) {
1132                             LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1133                                     + " is not a controller managed non intervpn link route. Ignoring.",
1134                                     vpn.getVrfId(), vrfEntry.getDestPrefix());
1135                             continue;
1136                         }
1137                         String prefix = vrfEntry.getDestPrefix();
1138                         String gwMac = vrfEntry.getGatewayMacAddress();
1139                         vrfEntry.getRoutePaths().forEach(routePath -> {
1140                             String nh = routePath.getNexthopAddress();
1141                             int label = routePath.getLabel().intValue();
1142                             if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1143                                     vrfEntry.getOrigin()))) {
1144                                 LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1145                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1146                                         vpnName, vpnRd);
1147                                 fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1148                                         Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1149                                         0 /*l3vni*/, gwMac,  null /*parentVpnRd*/, RouteOrigin.SELF_IMPORTED,
1150                                         writeConfigTxn);
1151                             } else {
1152                                 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} prefix {}"
1153                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1154                                         vpnName, vpnRd);
1155                                 SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1156                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, writeConfigTxn);
1157                             }
1158                         });
1159                     } catch (RuntimeException e) {
1160                         LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1161                                 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1162                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1163                     }
1164                 }
1165                 writeConfigTxn.submit();
1166             } else {
1167                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1168                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1169             }
1170         }
1171     }
1172
1173     @SuppressWarnings("checkstyle:IllegalCatch")
1174     @Override
1175     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1176         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1177         final String interfaceName = key.getName();
1178         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1179             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1180             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1181         }
1182     }
1183
1184     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1185                                 final VpnInterface vpnInterface, final String vpnName,
1186                                 final String interfaceName) {
1187         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1188             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1189                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1190                 deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn, vpnName);
1191                 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1192                 ListenableFuture<Void> futures = writeConfigTxn.submit();
1193                 String errorText = "removeVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
1194                         + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
1195                 ListenableFutures.addErrorLogging(futures, LOG, errorText);
1196                 return Collections.singletonList(futures);
1197             }, DJC_MAX_RETRIES);
1198         } else {
1199             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1200             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1201         }
1202     }
1203
1204     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1205     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1206                                 final VpnInterface vpnInterface, final String vpnName,
1207                                 final String interfaceName, final Interface interfaceState) {
1208         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1209                 vpnName, vpnInterface.getDpnId());
1210         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1211         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1212             () -> {
1213                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1214                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1215                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1216                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1217                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1218                                     interfaceName, vpnName);
1219                             BigInteger dpId = BigInteger.ZERO;
1220                             int ifIndex = 0;
1221                             String gwMacAddress = null;
1222                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1223                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1224                             final Optional<VpnInterfaceOpDataEntry> optVpnInterface =
1225                                     VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1226                             if (interfaceState != null) {
1227                                 try {
1228                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1229                                 } catch (NumberFormatException | IllegalStateException e) {
1230                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1231                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1232                                                     + " from vpn interface op data store. ", interfaceName,
1233                                             vpnInterface.getDpnId(), vpnName, e);
1234                                     dpId = BigInteger.ZERO;
1235                                 }
1236                                 ifIndex = interfaceState.getIfIndex();
1237                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1238                             } else {
1239                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1240                                         + " from vpn interface op.", interfaceName);
1241                                 if (optVpnInterface.isPresent()) {
1242                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1243                                     dpId = vpnOpInterface.getDpnId();
1244                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1245                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1246                                 } else {
1247                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1248                                                     + " as interfaceState and vpn interface op is not"
1249                                             + " available", interfaceName, vpnName);
1250                                     return;
1251                                 }
1252                             }
1253                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1254                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1255                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1256                             LOG.info(
1257                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1258                                             + "successfully",
1259                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1260                         })))));
1261                 futures.add(configFuture);
1262                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1263                 return futures;
1264             }, DJC_MAX_RETRIES);
1265     }
1266
1267     protected void processVpnInterfaceDown(BigInteger dpId,
1268                                            String interfaceName,
1269                                            int lportTag,
1270                                            String gwMac,
1271                                            VpnInterfaceOpDataEntry vpnOpInterface,
1272                                            boolean isInterfaceStateDown,
1273                                            WriteTransaction writeConfigTxn,
1274                                            WriteTransaction writeOperTxn,
1275                                            WriteTransaction writeInvTxn) {
1276         if (vpnOpInterface == null) {
1277             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1278                     + " as it is not available in operational data store", interfaceName, dpId);
1279             return;
1280         }
1281         final String vpnName = vpnOpInterface.getVpnInstanceName();
1282         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1283                                                     interfaceName, vpnName);
1284         if (!isInterfaceStateDown) {
1285             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1286             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1287                     null);
1288             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1289             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1290                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1291             if (interfaceManager.isExternalInterface(interfaceName)) {
1292                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1293                         NwConstants.DEL_FLOW);
1294             }
1295             if (!isBgpVpnInternetVpn) {
1296                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1297             }
1298             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1299                     + " successful", interfaceName, dpId, vpnName);
1300         } else {
1301             // Interface is retained in the DPN, but its Link Down.
1302             // Only withdraw the prefixes for this interface from BGP
1303             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1304         }
1305     }
1306
1307     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1308                                           final String vpnName, final long vpnId, String gwMac,
1309                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1310                                           final WriteTransaction writeInvTxn) {
1311         //Read NextHops
1312         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1313                 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1314         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1315         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1316
1317         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1318         LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1319                 interfaceName, dpnId, vpnName, primaryRd);
1320         if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1321             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1322             LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1323                     interfaceName, dpnId, vpnName, nextHops);
1324             for (Adjacency nextHop : nextHops) {
1325                 if (nextHop.isPhysNetworkFunc()) {
1326                     LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1327                             nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1328                     fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1329                             null/*writeCfgTxn*/);
1330                 } else {
1331                     String rd = nextHop.getVrfId();
1332                     List<String> nhList;
1333                     if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1334                         nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1335                     } else {
1336                         // This is a primary adjacency
1337                         nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1338                                 : Collections.emptyList();
1339                         removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1340                                 interfaceName, writeInvTxn);
1341                     }
1342                     if (!nhList.isEmpty()) {
1343                         if (rd.equals(vpnName)) {
1344                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1345                             //remove from FIB directly
1346                             nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1347                                     interfaceName, dpnId, writeConfigTxn));
1348                         } else {
1349                             removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, interfaceName,
1350                                     writeConfigTxn);
1351                         }
1352                     } else {
1353                         LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1354                                         + " interface {}", nextHop.getIpAddress(), rd,
1355                                 nextHop.getAdjacencyType().toString(), interfaceName);
1356                         bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1357                         fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1358                     }
1359                 }
1360                 String ip = nextHop.getIpAddress().split("/")[0];
1361                 LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1362                 if (vpnVipToPort != null) {
1363                     VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip);
1364                     LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1365                                     + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(),
1366                             ip, dpnId, vpnName);
1367                 }
1368             }
1369         } else {
1370             // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1371             LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1372                     + " Removing it.", interfaceName, vpnName, dpnId);
1373             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1374         }
1375     }
1376
1377     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1378                                                                    String interfaceName, BigInteger dpnId,
1379                                                                    WriteTransaction writeConfigTxn) {
1380         return (nh) -> {
1381             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1382                     writeConfigTxn);
1383             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1384                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1385                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1386         };
1387     }
1388
1389     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1390                                            BigInteger dpnId, String rd, String interfaceName,
1391                                            WriteTransaction writeConfigTxn) {
1392         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1393                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1394         nhList.forEach((nh) -> {
1395             //IRT: remove routes from other vpns importing it
1396             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1397                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1398             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1399                 String vpnRd = vpn.getVrfId();
1400                 if (vpnRd != null) {
1401                     fibManager.removeOrUpdateFibEntry(vpnRd,
1402                             nextHop.getIpAddress(), nh, writeConfigTxn);
1403                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1404                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1405                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1406                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1407                 }
1408             }
1409         });
1410     }
1411
1412     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1413                                                  int lportTag, String gwMac, String interfaceName,
1414                                                  WriteTransaction writeInvTxn) {
1415         final Uuid subnetId = nextHop.getSubnetId();
1416         if (nextHop.getSubnetGatewayMacAddress() == null) {
1417             // A valid mac-address was not available for this subnet-gateway-ip
1418             // So a connected-mac-address was used for this subnet and we need
1419             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1420             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1421                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1422         }
1423         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1424                 subnetId);
1425     }
1426
1427     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1428                                                   String interfaceName) {
1429         // This is either an extra-route (or) a learned IP via subnet-route
1430         List<String> nhList = null;
1431         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1432         if (nextHopIp == null || nextHopIp.isEmpty()) {
1433             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1434                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1435                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1436                     vpnName);
1437             nhList = Collections.emptyList();
1438         } else {
1439             nhList = Collections.singletonList(nextHopIp);
1440         }
1441         return nhList;
1442     }
1443
1444     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1445         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1446         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1447         // else use connected interface
1448         if (gwPort != null && gwPort.isSubnetIp()) {
1449             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1450                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1451             return Optional.of(gwPort.getMacAddress());
1452         }
1453         return Optional.absent();
1454     }
1455
1456     @Override
1457     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1458         final VpnInterface update) {
1459         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1460                 update.getDpnId(), original.getVpnInstanceNames(),
1461                 update.getVpnInstanceNames());
1462         final String vpnInterfaceName = update.getName();
1463         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1464         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1465         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1466             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1467         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1468         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1469             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1470
1471         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1472         //handles switching between <internal VPN - external VPN>
1473         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1474             LOG.info("update: handled VPNInterface {} on dpn {} update"
1475                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1476                      original.getName(), dpnId,
1477                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1478                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1479             return;
1480         }
1481         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1482             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1483             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1484             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1485             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1486             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1487                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1488                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1489                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1490                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1491                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1492                     LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1493                             update.getName(), newVpnName);
1494                     //handle both addition and removal of adjacencies
1495                     //currently, new adjacency may be an extra route
1496                     boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1497                     if (!oldAdjs.equals(newAdjs)) {
1498                         for (Adjacency adj : copyNewAdjs) {
1499                             if (copyOldAdjs.contains(adj)) {
1500                                 copyOldAdjs.remove(adj);
1501                             } else {
1502                                 // add new adjacency - right now only extra route will hit this path
1503                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1504                                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1505                                              dpnId, writeOperTxn, writeConfigTxn);
1506                                 }
1507                                 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to vpn "
1508                                                 + "interface {} on vpn {} dpnId {}",
1509                                         adj.getIpAddress(), adj.getNextHopIpList(),
1510                                         adj.getLabel(), adj.getSubnetId(), update.getName(),
1511                                         newVpnName, dpnId);
1512                             }
1513                         }
1514                         for (Adjacency adj : copyOldAdjs) {
1515                             if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1516                                 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1517                                        writeOperTxn, writeConfigTxn);
1518                             }
1519                             LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1520                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1521                                 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1522                         }
1523                     }
1524                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
1525                     try {
1526                         operFuture.get();
1527                     } catch (ExecutionException e) {
1528                         LOG.error("Exception encountered while submitting operational future for update"
1529                                 + " VpnInterface {} on vpn {}", vpnInterfaceName, newVpnName, e);
1530                         return null;
1531                     }
1532                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1533                     futures.add(writeConfigTxn.submit());
1534                     LOG.info("update: vpn interface updated for interface {} oldVpn(s) {} newVpn {}"
1535                         + "processed successfully", update.getName(),
1536                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), newVpnName);
1537                     return futures;
1538                 });
1539             } else {
1540                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1541                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1542             }
1543         }
1544     }
1545
1546     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1547                                                  VpnInterface original, VpnInterface update) {
1548         boolean isSwap = Boolean.FALSE;
1549         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1550         final String interfaceName = key.getName();
1551         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1552             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1553         List<String> oldVpnListCopy = new ArrayList<>();
1554         oldVpnListCopy.addAll(oldVpnList);
1555         List<String> newVpnList = update.getVpnInstanceNames().stream()
1556             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1557         oldVpnList.removeAll(newVpnList);
1558         newVpnList.removeAll(oldVpnListCopy);
1559         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1560             for (String oldVpnName: oldVpnList) {
1561                 isSwap = Boolean.TRUE;
1562                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1563                         + " running config-driven swap removal", interfaceName, oldVpnName);
1564                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1565                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1566                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1567             }
1568             //Wait for previous interface bindings to be removed
1569             try {
1570                 Thread.sleep(2000);
1571             } catch (InterruptedException e) {
1572                 //Ignore
1573             }
1574             for (String newVpnName: newVpnList) {
1575                 String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1576                 isSwap = Boolean.TRUE;
1577                 if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1578                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1579                             + "running config-driven swap addition", interfaceName, newVpnName);
1580                     final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1581                     final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1582                             ? origAdjs.getAdjacency() : new ArrayList<>();
1583                     final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1584                     final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1585                             ? updateAdjs.getAdjacency() : new ArrayList<>();
1586
1587                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1588                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1589                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1590                             interfaceName, oldVpnListCopy, newVpnName);
1591                 }
1592             }
1593         }
1594         return isSwap;
1595     }
1596
1597     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1598         Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1599         synchronized (label.toString().intern()) {
1600             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1601                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1602             Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1603             if (opResult.isPresent()) {
1604                 LabelRouteInfo labelRouteInfo =
1605                     new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1606                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1607             }
1608         }
1609         LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList);
1610     }
1611
1612     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1613         SubnetRoute route, WriteTransaction writeConfigTxn) {
1614
1615         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1616         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, null /* parentVpnRd */)
1617                 .addAugmentation(SubnetRoute.class, route).build();
1618         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1619         InstanceIdentifierBuilder<VrfTables> idBuilder =
1620             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1621         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1622         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1623         if (writeConfigTxn != null) {
1624             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1625         } else {
1626             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1627         }
1628         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1629                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1630     }
1631
1632     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1633                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1634                                            WriteTransaction writeConfigTxn) {
1635
1636         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1637                                                 LogicalDatastoreType.OPERATIONAL, identifier);
1638
1639         if (optVpnInterface.isPresent()) {
1640             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1641             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1642             String vpnName = currVpnIntf.getVpnInstanceName();
1643             VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1644             InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1645             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker,
1646                                             LogicalDatastoreType.OPERATIONAL, adjPath);
1647             boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1648             VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1649             long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1650             VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1651             List<Adjacency> adjacencies;
1652             if (optAdjacencies.isPresent()) {
1653                 adjacencies = optAdjacencies.get().getAdjacency();
1654             } else {
1655                 // This code will be hit in case of first PNF adjacency
1656                 adjacencies = new ArrayList<>();
1657             }
1658             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1659             L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1660                     .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1661             Adjacency operationalAdjacency = null;
1662             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) {
1663                 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1664                         : RouteOrigin.STATIC;
1665                 String nh = adj.getNextHopIpList().get(0);
1666                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1667                 synchronized (vpnPrefixKey.intern()) {
1668                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1669                             dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1670                     if (rdToAllocate.isPresent()) {
1671                         input.setRd(rdToAllocate.get());
1672                         operationalAdjacency = populator.createOperationalAdjacency(input);
1673                         int label = operationalAdjacency.getLabel().intValue();
1674                         vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1675                                 currVpnIntf.getVpnInstanceName(), l3vni, origin,
1676                                 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1677                         LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1678                                 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1679                                 vpnName, label, currVpnIntf.getName(), dpnId);
1680                     } else {
1681                         LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName,
1682                                 prefix);
1683                         return;
1684                     }
1685                     // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1686                     // Keeping the MPLS check for now.
1687                     if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1688                         final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1689                         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1690                                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1691                         vpnsToImportRoute.forEach(vpn -> {
1692                             if (vpn.getVrfId() != null) {
1693                                 VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1694                                         dataBroker, vpn.getVpnId(), vpnId, prefix,
1695                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1696                                         .ifPresent(
1697                                             rds -> vpnManager.addExtraRoute(
1698                                                     VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1699                                                     adj.getIpAddress(), nh, rds,
1700                                                     currVpnIntf.getVpnInstanceName(),
1701                                                     l3vni, RouteOrigin.SELF_IMPORTED,
1702                                                     currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn));
1703                             }
1704                         });
1705                     }
1706                 }
1707             } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1708                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1709                         currVpnIntf.getName(), vpnName);
1710
1711                 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1712
1713                 writeOperTxn.merge(
1714                         LogicalDatastoreType.OPERATIONAL,
1715                         VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1716                                 adj.getSubnetId().getValue()), prefix),
1717                         VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1718                                 adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1719
1720                 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1721                         adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, 0 /*l3vni*/,
1722                       null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1723
1724                 input.setRd(adj.getVrfId());
1725             }
1726             if (operationalAdjacency == null) {
1727                 operationalAdjacency = populator.createOperationalAdjacency(input);
1728             }
1729             adjacencies.add(operationalAdjacency);
1730             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1731             VpnInterfaceOpDataEntry newVpnIntf =
1732                 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1733                     aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1734                         currVpnIntf.getGatewayMacAddress());
1735
1736             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1737         }
1738     }
1739
1740     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1741         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1742         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1743     }
1744
1745     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1746             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1747         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1748                                           LogicalDatastoreType.OPERATIONAL, identifier);
1749
1750         if (optVpnInterface.isPresent()) {
1751             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1752
1753             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1754             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1755             if (optAdjacencies.isPresent()) {
1756                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1757
1758                 if (!adjacencies.isEmpty()) {
1759                     LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1760                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1761                     while (adjIt.hasNext()) {
1762                         Adjacency adjElem = adjIt.next();
1763                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1764                             String rd = adjElem.getVrfId();
1765                             adjIt.remove();
1766
1767                             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1768                             VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1769                                     .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1770                                     currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove(),
1771                                             currVpnIntf.getLportTag(), currVpnIntf.getGatewayMacAddress());
1772
1773                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1774                             if (adj.getNextHopIpList() != null) {
1775                                 for (String nh : adj.getNextHopIpList()) {
1776                                     deleteExtraRouteFromCurrentAndImportingVpns(
1777                                         currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1778                                         currVpnIntf.getName(), writeConfigTxn);
1779                                 }
1780                             } else if (adj.isPhysNetworkFunc()) {
1781                                 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1782                                         adj.getIpAddress(), adj.getSubnetId());
1783                                 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1784                                         writeConfigTxn);
1785                             }
1786                             break;
1787                         }
1788
1789                     }
1790                 }
1791                 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1792                         dpnId, adj.getVrfId());
1793             } else {
1794                 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1795                         + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1796             }
1797         }
1798     }
1799
1800     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1801         String rd, String intfName, WriteTransaction writeConfigTxn) {
1802         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1803         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1804         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1805             String vpnRd = vpn.getVrfId();
1806             if (vpnRd != null) {
1807                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1808             }
1809         }
1810     }
1811
1812     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1813         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1814             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1815             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1816     }
1817
1818     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1819         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1820             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1821     }
1822
1823     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1824         WriteTransaction writeOperTxn) {
1825         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1826         if (dpId.equals(BigInteger.ZERO)) {
1827             LOG.error("addToNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1828                     + " association model", vpnInterfaceName, routerName);
1829
1830             return;
1831         }
1832         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1833
1834         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1835             .OPERATIONAL, routerDpnListIdentifier);
1836         RouterInterfaces routerInterface =
1837             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1838                 vpnInterfaceName).build();
1839         if (optionalRouterDpnList.isPresent()) {
1840             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1841                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1842         } else {
1843             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1844             builder.setRouterId(routerName);
1845             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1846             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
1847             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1848                 getRouterId(routerName),
1849                 builder.build(), true);
1850         }
1851     }
1852
1853     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1854         WriteTransaction writeOperTxn) {
1855         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1856         if (dpId.equals(BigInteger.ZERO)) {
1857             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1858                     + " dissociation model", vpnInterfaceName, routerName);
1859
1860             return;
1861         }
1862         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1863         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1864             .OPERATIONAL, routerDpnListIdentifier);
1865         if (optionalRouterDpnList.isPresent()) {
1866             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1867             RouterInterfaces routerInterface =
1868                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1869                     vpnInterfaceName).build();
1870
1871             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1872                 if (routerInterfaces.isEmpty()) {
1873                     if (writeOperTxn != null) {
1874                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1875                     } else {
1876                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1877                     }
1878                 } else {
1879                     if (writeOperTxn != null) {
1880                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1881                             RouterInterfaces.class,
1882                             new RouterInterfacesKey(vpnInterfaceName)));
1883                     } else {
1884                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1885                             routerDpnListIdentifier.child(
1886                                 RouterInterfaces.class,
1887                                 new RouterInterfacesKey(vpnInterfaceName)));
1888                     }
1889                 }
1890             }
1891         }
1892     }
1893
1894     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1895         WriteTransaction writeOperTxn) {
1896         if (dpId.equals(BigInteger.ZERO)) {
1897             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1898                     + " dissociation model", vpnInterfaceName, routerName);
1899
1900             return;
1901         }
1902         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1903         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1904             .OPERATIONAL, routerDpnListIdentifier);
1905         if (optionalRouterDpnList.isPresent()) {
1906             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1907             RouterInterfaces routerInterface =
1908                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1909                     vpnInterfaceName).build();
1910             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1911                 if (routerInterfaces.isEmpty()) {
1912                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1913                 } else {
1914                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1915                         RouterInterfaces.class,
1916                         new RouterInterfacesKey(vpnInterfaceName)));
1917                 }
1918             }
1919         }
1920     }
1921
1922     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1923                                                     WriteTransaction writeConfigTxn, String vpnName) {
1924         if (vpnInterface == null) {
1925             return;
1926         }
1927         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1928         if (adjs == null) {
1929             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1930                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1931             return;
1932         }
1933         for (Adjacency adj : adjs) {
1934             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1935                 String primaryInterfaceIp = adj.getIpAddress();
1936                 String macAddress = adj.getMacAddress();
1937                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1938
1939                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1940                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1941
1942                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1943                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1944                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1945                         routerInt, label, writeConfigTxn);
1946                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1947                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1948                         macAddress);
1949                 return;
1950             }
1951         }
1952         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1953                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1954                 primaryRd, vpnName);
1955     }
1956
1957     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1958                                                     WriteTransaction writeConfigTxn, String vpnName) {
1959         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
1960         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1961         if (adjs != null) {
1962             List<Adjacency> adjsList = adjs.getAdjacency();
1963             for (Adjacency adj : adjsList) {
1964                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1965                     String primaryInterfaceIp = adj.getIpAddress();
1966                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1967                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1968                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1969                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1970                     return;
1971                 }
1972             }
1973         } else {
1974             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1975                     vpnInterface.getName(), rd);
1976         }
1977     }
1978
1979     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1980         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1981             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1982                   vpnName, intefaceData.vpnInterface.getName());
1983             return;
1984         }
1985         final VpnInterfaceKey key = intefaceData.identifier
1986                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1987         final String interfaceName = key.getName();
1988         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1989                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1990         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1991                   intefaceData.identifier, vpnName);
1992         return;
1993     }
1994
1995     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1996                                               VpnInterface vpnInterface, String vpnName) {
1997         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
1998                .get(vpnName);
1999         if (vpnInterfaces == null) {
2000             vpnInterfaces = new ConcurrentLinkedQueue<>();
2001         }
2002         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2003         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2004         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2005                 vpnInterface.getName(), vpnName);
2006     }
2007
2008     public boolean isVpnInstanceReady(String vpnInstanceName) {
2009         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2010         if (vpnRd == null) {
2011             return false;
2012         }
2013         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2014
2015         return vpnInstanceOpDataEntry != null;
2016     }
2017
2018     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2019         synchronized (vpnInstanceName.intern()) {
2020             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2021                     unprocessedVpnInterfaces.get(vpnInstanceName);
2022             if (vpnInterfaces != null) {
2023                 while (!vpnInterfaces.isEmpty()) {
2024                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2025                     if (hasVpnInstanceCreatedSuccessfully) {
2026                         processSavedInterface(savedInterface, vpnInstanceName);
2027                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2028                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2029                     } else {
2030                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2031                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2032                     }
2033                 }
2034             } else {
2035                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2036             }
2037         }
2038     }
2039
2040     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2041             VpnInterface vpnInterface) {
2042         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2043             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2044                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2045             if (vpnInterfaces != null) {
2046                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2047                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2048                             + "unprocessed list", vpnInterface.getName(),
2049                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2050                 }
2051             } else {
2052                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2053                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2054             }
2055         }
2056     }
2057
2058     public void vpnInstanceIsReady(String vpnInstanceName) {
2059         processSavedInterfaces(vpnInstanceName, true);
2060     }
2061
2062     public void vpnInstanceFailed(String vpnInstanceName) {
2063         processSavedInterfaces(vpnInstanceName, false);
2064     }
2065
2066     private static class UnprocessedVpnInterfaceData {
2067         InstanceIdentifier<VpnInterface> identifier;
2068         VpnInterface vpnInterface;
2069
2070         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2071             this.identifier = identifier;
2072             this.vpnInterface = vpnInterface;
2073         }
2074
2075         @Override
2076         public int hashCode() {
2077             final int prime = 31;
2078             int result = 1;
2079             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2080             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2081             return result;
2082         }
2083
2084         @Override
2085         public boolean equals(Object obj) {
2086             if (this == obj) {
2087                 return true;
2088             }
2089             if (obj == null) {
2090                 return false;
2091             }
2092             if (getClass() != obj.getClass()) {
2093                 return false;
2094             }
2095             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2096             if (identifier == null) {
2097                 if (other.identifier != null) {
2098                     return false;
2099                 }
2100             } else if (!identifier.equals(other.identifier)) {
2101                 return false;
2102             }
2103             if (vpnInterface == null) {
2104                 if (other.vpnInterface != null) {
2105                     return false;
2106                 }
2107             } else if (!vpnInterface.equals(other.vpnInterface)) {
2108                 return false;
2109             }
2110             return true;
2111         }
2112     }
2113
2114     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2115         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2116         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2117         if (vpnInstanceOpData == null) {
2118             return;
2119         }
2120         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2121         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2122             return;
2123         }
2124         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2125         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2126             InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2127                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2128             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = VpnUtil.read(dataBroker,
2129                     LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2130             if (!vpnInterfaceOptional.isPresent()) {
2131                 return;
2132             }
2133             List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2134                     vpnInterface.getInterfaceName());
2135             if (configVpnAdjacencies == null) {
2136                 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2137                 return;
2138             }
2139             List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2140                     .getAugmentation(AdjacenciesOp.class).getAdjacency();
2141             // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2142             // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2143             // in operational DS. These unprocessed adjacencies will be handled below.
2144             // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2145             // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2146             configVpnAdjacencies.stream()
2147                     .filter(adjacency -> operationVpnAdjacencies.stream()
2148                             .noneMatch(operationalAdjacency ->
2149                                     operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2150                     .forEach(adjacency -> {
2151                         LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2152                         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2153                             () -> {
2154                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
2155                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
2156                                 if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2157                                     addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd, adjacency,
2158                                             vpnInterfaceOptional.get().getDpnId(), writeConfigTxn, writeOperTxn);
2159                                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
2160                                     try {
2161                                         operFuture.get();
2162                                     } catch (ExecutionException | InterruptedException e) {
2163                                         LOG.error("Exception encountered while submitting operational"
2164                                                 + " future for vpnInterface {}", vpnInterface, e);
2165                                     }
2166                                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2167                                     futures.add(writeConfigTxn.submit());
2168                                     return futures;
2169                                 }  else {
2170                                     return Collections.emptyList();
2171                                 }
2172                             });
2173                     });
2174         }));
2175     }
2176
2177     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2178         private final String interfaceName;
2179         private final boolean add;
2180         private final String txnDestination;
2181
2182         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2183             this.interfaceName = interfaceName;
2184             this.add = add;
2185             this.txnDestination = transactionDest;
2186         }
2187
2188         @Override
2189         public void onSuccess(Void voidObj) {
2190             if (add) {
2191                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2192                         interfaceName, txnDestination);
2193             } else {
2194                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2195             }
2196         }
2197
2198         @Override
2199         public void onFailure(Throwable throwable) {
2200             if (add) {
2201                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2202                         interfaceName, txnDestination, throwable);
2203             } else {
2204                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2205                 VpnUtil.unsetScheduledToRemoveForVpnInterface(txRunner, interfaceName);
2206             }
2207         }
2208     }
2209 }