vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.concurrent.ConcurrentHashMap;
  25 import java.util.concurrent.ConcurrentLinkedQueue;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34
  35 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  36 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  37 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  41 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  42 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  43 import org.opendaylight.genius.mdsalutil.NWUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.cache.DataObjectCache;
  46 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  47 import org.opendaylight.infrautils.caches.CacheProvider;
  48 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  49 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  50 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  51 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  52 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  53 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  54 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  55 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  56 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  57 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  58 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  59 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  60 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 110 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 111 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 112 import org.slf4j.Logger;
 113 import org.slf4j.LoggerFactory;
 114
 115 @Singleton
 116 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 117
 118     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 119     private static final short DJC_MAX_RETRIES = 3;
 120
 121     private final DataBroker dataBroker;
 122     private final ManagedNewTransactionRunner txRunner;
 123     private final IBgpManager bgpManager;
 124     private final IFibManager fibManager;
 125     private final IMdsalApiManager mdsalManager;
 126     private final IdManagerService idManager;
 127     private final OdlInterfaceRpcService ifaceMgrRpcService;
 128     private final VpnFootprintService vpnFootprintService;
 129     private final IInterfaceManager interfaceManager;
 130     private final IVpnManager vpnManager;
 131     private final ArpResponderHandler arpResponderHandler;
 132     private final JobCoordinator jobCoordinator;
 133
 134     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 135
 136     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 137             new ConcurrentHashMap<>();
 138
 139     private final DataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 140
 141     @Inject
 142     public VpnInterfaceManager(final DataBroker dataBroker,
 143                                final IBgpManager bgpManager,
 144                                final IdManagerService idManager,
 145                                final IMdsalApiManager mdsalManager,
 146                                final IFibManager fibManager,
 147                                final OdlInterfaceRpcService ifaceMgrRpcService,
 148                                final VpnFootprintService vpnFootprintService,
 149                                final IInterfaceManager interfaceManager,
 150                                final IVpnManager vpnManager,
 151                                final ArpResponderHandler arpResponderHandler,
 152                                final JobCoordinator jobCoordinator,
 153                                final CacheProvider cacheProvider) {
 154         super(VpnInterface.class, VpnInterfaceManager.class);
 155
 156         this.dataBroker = dataBroker;
 157         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 158         this.bgpManager = bgpManager;
 159         this.idManager = idManager;
 160         this.mdsalManager = mdsalManager;
 161         this.fibManager = fibManager;
 162         this.ifaceMgrRpcService = ifaceMgrRpcService;
 163         this.vpnFootprintService = vpnFootprintService;
 164         this.interfaceManager = interfaceManager;
 165         this.vpnManager = vpnManager;
 166         this.arpResponderHandler = arpResponderHandler;
 167         this.jobCoordinator = jobCoordinator;
 168
 169         vpnInstanceOpDataEntryCache = new DataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 170                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 171                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 172     }
 173
 174     public Runnable isNotifyTaskQueued(String intfName) {
 175         return vpnIntfMap.remove(intfName);
 176     }
 177
 178     @PostConstruct
 179     public void start() {
 180         LOG.info("{} start", getClass().getSimpleName());
 181         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 182     }
 183
 184     @Override
 185     @PreDestroy
 186     public void close() {
 187         super.close();
 188         vpnInstanceOpDataEntryCache.close();
 189     }
 190
 191     @Override
 192     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 193         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 194     }
 195
 196     @Override
 197     protected VpnInterfaceManager getDataTreeChangeListener() {
 198         return VpnInterfaceManager.this;
 199     }
 200
 201     @Override
 202     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 203         LOG.info("add: intfName {} onto vpnName {}",
 204                  vpnInterface.getName(),
 205                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 206         addVpnInterface(identifier, vpnInterface, null, null);
 207     }
 208
 209     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 210                           final VpnInterface vpnInterface, String vpnName) {
 211         synchronized (vpnName.intern()) {
 212             if (isVpnInstanceReady(vpnName)) {
 213                 return true;
 214             }
 215             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 216             return false;
 217         }
 218     }
 219
 220     // TODO Clean up the exception handling
 221     @SuppressWarnings("checkstyle:IllegalCatch")
 222     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 223                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 224         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 225             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 226             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 227         }
 228     }
 229
 230     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 231                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 232         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 233         final String interfaceName = key.getName();
 234
 235         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 236             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 237                   vpnName, vpnInterface.getName());
 238             return;
 239         }
 240         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 241              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 242         List<Adjacency> copyOldAdjs = null;
 243         if (oldAdjs != null) {
 244             copyOldAdjs = new ArrayList<>();
 245             copyOldAdjs.addAll(oldAdjs);
 246         }
 247         List<Adjacency> copyNewAdjs = null;
 248         if (newAdjs != null) {
 249             copyNewAdjs = new ArrayList<>();
 250             copyNewAdjs.addAll(newAdjs);
 251         }
 252         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 253     }
 254
 255     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 256                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 257                     final List<Adjacency> newAdjs,
 258                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 259         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 260         final String interfaceName = key.getName();
 261         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 262         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 263             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 264             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 265             if (interfaceState != null) {
 266                 try {
 267                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 268                     final int ifIndex = interfaceState.getIfIndex();
 269                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 270                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 271                         WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 272                         WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 273                         LOG.info("addVpnInterface: VPN Interface add event - intfName {} vpnName {} on dpn {}" ,
 274                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 275                         processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, writeConfigTxn,
 276                                      writeOperTxn, writeInvTxn, interfaceState, vpnName);
 277                         if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 278                             LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 279                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 280                             if (newAdjs != null) {
 281                                 for (Adjacency adj : newAdjs) {
 282                                     if (oldAdjs.contains(adj)) {
 283                                         oldAdjs.remove(adj);
 284                                     } else {
 285                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 286                                                                   dataBroker, adj)) {
 287                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd,
 288                                                   adj, dpnId, writeOperTxn, writeConfigTxn);
 289                                         }
 290                                     }
 291                                 }
 292                             }
 293                             for (Adjacency adj : oldAdjs) {
 294                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 295                                                           dataBroker, adj)) {
 296                                     delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 297                                         writeOperTxn, writeConfigTxn);
 298                                 }
 299                             }
 300                         }
 301                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
 302                         try {
 303                             operFuture.get();
 304                         } catch (ExecutionException e) {
 305                             LOG.error("addVpnInterface: Exception encountered while submitting operational future for"
 306                                     + " addVpnInterface {} on vpn {}", vpnInterface.getName(), vpnName, e);
 307                             return null;
 308                         }
 309                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 310                         ListenableFuture<Void> configFuture = writeConfigTxn.submit();
 311                         futures.add(configFuture);
 312                         Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"));
 313                         futures.add(writeInvTxn.submit());
 314                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 315                                 + " processed successfully", interfaceName, vpnName, dpnId);
 316                         return futures;
 317                     });
 318                 } catch (NumberFormatException | IllegalStateException e) {
 319                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 320                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 321                                     vpnName, e);
 322                     return;
 323                 }
 324             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 325                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 326                     () -> {
 327                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 328                         createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 329                                        writeConfigTxn, vpnName);
 330                         LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 331                                  vpnName, vpnInterface.getDpnId());
 332                         ListenableFuture<Void> futures = writeConfigTxn.submit();
 333                         String errorText = "addVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
 334                              + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
 335                         ListenableFutures.addErrorLogging(futures, LOG, errorText);
 336                         return Collections.singletonList(futures);
 337                     });
 338             } else {
 339                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 340                         + " is not available", interfaceName, vpnName);
 341             }
 342         } else {
 343             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 344                       + " as vpn is pending delete", interfaceName, vpnName,
 345                     vpnInterface.getDpnId());
 346         }
 347     }
 348
 349     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 350     // see comments below.
 351     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 352     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 353             final int lportTag, boolean isInterfaceUp,
 354             WriteTransaction writeConfigTxn,
 355             WriteTransaction writeOperTxn,
 356             WriteTransaction writeInvTxn,
 357             Interface interfaceState,
 358             final String vpnName) {
 359         final String interfaceName = vpnInterface.getName();
 360         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 361                                                          interfaceName, vpnName);
 362         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 363         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 364         if (!isInterfaceUp) {
 365             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 366                      interfaceName, dpId, vpnName);
 367             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 368             if (vpnId == VpnConstants.INVALID_ID) {
 369                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 370                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 371                         dpId);
 372                 return;
 373             }
 374
 375             boolean waitForVpnInterfaceOpRemoval = false;
 376             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 377                 String opVpnName = opVpnInterface.getVpnInstanceName();
 378                 String primaryInterfaceIp = null;
 379                 if (opVpnName.equals(vpnName)) {
 380                     // Please check if the primary VRF Entry does not exist for VPNInterface
 381                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 382                     // back to back
 383                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 384                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 385                     if (adjs == null) {
 386                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 387                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 388                                 vpnName);
 389                         return;
 390                     }
 391                     for (Adjacency adj : adjs) {
 392                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 393                             primaryInterfaceIp = adj.getIpAddress();
 394                             break;
 395                         }
 396                     }
 397                     if (primaryInterfaceIp == null) {
 398                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 399                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 400                                 dpId, vpnName);
 401                         return;
 402                     }
 403                     // Get the rd of the vpn instance
 404                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 405                     if (vrf != null) {
 406                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 407                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 408                         return;
 409                     }
 410                     waitForVpnInterfaceOpRemoval = true;
 411                 } else {
 412                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 413                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 414                 }
 415             }
 416             if (!waitForVpnInterfaceOpRemoval) {
 417                 // Add the VPNInterface and quit
 418                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 419                         null/*ipAddressSourceValuePair*/,
 420                         true /* add */);
 421                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 422                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 423                 if (!isBgpVpnInternetVpn) {
 424                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 425                             jobCoordinator);
 426                 }
 427                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 428                         dpId, vpnName);
 429                 if (interfaceManager.isExternalInterface(interfaceName)) {
 430                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 431                         NwConstants.ADD_FLOW);
 432                 }
 433                 return;
 434             }
 435
 436             // FIB didn't get a chance yet to clean up this VPNInterface
 437             // Let us give it a chance here !
 438             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 439                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 440             try {
 441                 Runnable notifyTask = new VpnNotifyTask();
 442                 synchronized (notifyTask) {
 443                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 444                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 445                     // published here while holding the lock on it so this path will hit the wait before notify can be
 446                     // invoked.
 447                     vpnIntfMap.put(interfaceName, notifyTask);
 448                     try {
 449                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 450                     } catch (InterruptedException e) {
 451                         // Ignored
 452                     }
 453                 }
 454             } finally {
 455                 vpnIntfMap.remove(interfaceName);
 456             }
 457
 458             if (opVpnInterface != null) {
 459                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 460                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 461                         dpId, vpnName);
 462                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
 463                 return;
 464             }
 465             // VPNInterface got removed, proceed with Add
 466             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 467                     interfaceName, dpId, vpnName);
 468             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 469                     null/*ipAddressSourceValuePair*/,
 470                     true /* add */);
 471             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 472                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 473             if (!isBgpVpnInternetVpn) {
 474                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 475             }
 476             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 477                     + " FIB to clean up", interfaceName, dpId, vpnName);
 478             if (interfaceManager.isExternalInterface(interfaceName)) {
 479                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 480                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 481             }
 482         } else {
 483             // Interface is retained in the DPN, but its Link Up.
 484             // Advertise prefixes again for this interface to BGP
 485             InstanceIdentifier<VpnInterface> identifier =
 486                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 487             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 488                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 489             advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 490             // Perform similar operation as interface add event for extraroutes.
 491             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 492             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 493             if (!optAdjacencies.isPresent()) {
 494                 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 495                 return;
 496             }
 497             List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 498             for (Adjacency adjacency : adjacencies) {
 499                 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 500                     continue;
 501                 }
 502                 // if BGPVPN Internet, filter only IPv6 Adjacencies
 503                 if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 504                                 dataBroker, adjacency)) {
 505                     continue;
 506                 }
 507                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 508                           dpId, writeOperTxn, writeConfigTxn);
 509             }
 510         }
 511     }
 512
 513     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 514             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 515         Uuid extNetworkId;
 516         try {
 517             // vpn instance of ext-net interface is the network-id
 518             extNetworkId = new Uuid(vpnName);
 519         } catch (IllegalArgumentException e) {
 520             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 521                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 522             return;
 523         }
 524
 525         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 526         if (routerIds == null || routerIds.isEmpty()) {
 527             LOG.info("processExternalVpnInterface: No router is associated with {}."
 528                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 529                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 530             return;
 531         }
 532
 533         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 534                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 535         for (Uuid routerId : routerIds) {
 536             String routerName = routerId.getValue();
 537             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 538             if (Objects.equals(primarySwitch, dpId)) {
 539                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 540                 if (router != null) {
 541                     if (addOrRemove == NwConstants.ADD_FLOW) {
 542                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 543                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 544                                 dpId, interfaceName, lportTag, writeInvTxn);
 545                     } else {
 546                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 547                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 548                                 dpId, interfaceName, lportTag);
 549                     }
 550                 } else {
 551                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 552                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 553                             interfaceName, dpId, vpnName);
 554                 }
 555             }
 556         }
 557     }
 558
 559     // TODO Clean up the exception handling
 560     @SuppressWarnings("checkstyle:IllegalCatch")
 561     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 562                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 563                                                   String vpnName, String interfaceName) {
 564         if (rd == null) {
 565             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 566                   interfaceName, dpnId, vpnName);
 567             return;
 568         } else {
 569             if (rd.equals(vpnName)) {
 570                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 571                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 572                 return;
 573             }
 574         }
 575         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 576                 interfaceName, dpnId, vpnName, rd);
 577
 578         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 579         if (nextHopIp == null) {
 580             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 581                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 582                     rd, vpnName);
 583             return;
 584         }
 585
 586         //Read NextHops
 587         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 588         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 589         if (adjacencies.isPresent()) {
 590             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 591
 592             if (!nextHops.isEmpty()) {
 593                 LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 594                         + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 595                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 596                 long l3vni = vpnInstanceOpData.getL3vni();
 597                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 598                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 599                 for (Adjacency nextHop : nextHops) {
 600                     if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 601                         continue;
 602                     }
 603                     String gatewayMac = null;
 604                     long label = 0;
 605                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 606                         final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 607                             vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 608                         gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName).get();
 609                     } else {
 610                         label = nextHop.getLabel();
 611                     }
 612                     try {
 613                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 614                                 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 615                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 616                                 encapType, (int)label, l3vni, 0 /*l2vni*/,
 617                                 gatewayMac);
 618                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 619                                 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 620                                 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 621                     } catch (Exception e) {
 622                         LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}"
 623                                 + " for interface {} on dpn {}", nextHop.getIpAddress(),
 624                                   vpnName, rd, interfaceName, dpnId, e);
 625                     }
 626                 }
 627             }
 628         }
 629     }
 630
 631     // TODO Clean up the exception handling
 632     @SuppressWarnings("checkstyle:IllegalCatch")
 633     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 634                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 635         //Read NextHops
 636         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 637         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 638
 639         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 640         if (rd == null) {
 641             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 642                 interfaceName, vpnName);
 643             return;
 644         } else {
 645             if (rd.equals(vpnName)) {
 646                 LOG.info(
 647                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 648                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 649                 return;
 650             }
 651         }
 652         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 653                vpnName, rd);
 654         if (adjacencies.isPresent()) {
 655             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 656
 657             if (!nextHops.isEmpty()) {
 658                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 659                         nextHops, interfaceName, vpnName, rd);
 660                 for (Adjacency nextHop : nextHops) {
 661                     try {
 662                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 663                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 664                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 665                                     interfaceName, vpnName);
 666                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 667                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 668                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 669                                     interfaceName, vpnName);
 670                         } else {
 671                             // Perform similar operation as interface delete event for extraroutes.
 672                             String allocatedRd = nextHop.getVrfId();
 673                             for (String nh : nextHop.getNextHopIpList()) {
 674                                 deleteExtraRouteFromCurrentAndImportingVpns(
 675                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 676                             }
 677                         }
 678                     } catch (Exception e) {
 679                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 680                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 681                     }
 682                 }
 683             }
 684         }
 685     }
 686
 687     @SuppressWarnings("checkstyle:IllegalCatch")
 688     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 689                                                   String primaryRd, String interfaceName, final long vpnId,
 690                                                   WriteTransaction writeConfigTxn,
 691                                                   WriteTransaction writeOperTxn,
 692                                                   final WriteTransaction writeInvTxn,
 693                                                   Interface interfaceState) {
 694         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 695         // Read NextHops
 696         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 697         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 698         if (!adjacencies.isPresent()) {
 699             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 700                     null/*gwMac*/, writeOperTxn);
 701             return;
 702         }
 703
 704         // Get the rd of the vpn instance
 705         String nextHopIp = null;
 706         try {
 707             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 708         } catch (Exception e) {
 709             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 710                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 711         }
 712         List<String> nhList = new ArrayList<>();
 713         if (nextHopIp != null) {
 714             nhList.add(nextHopIp);
 715             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 716                     interfaceName, dpnId, vpnName, nhList);
 717         }
 718         Optional<String> gwMac = Optional.absent();
 719         String vpnInterfaceSubnetGwMacAddress = null;
 720         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 721         Long l3vni = vpnInstanceOpData.getL3vni();
 722         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 723         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 724         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 725         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 726         List<Adjacency> value = new ArrayList<>();
 727         for (Adjacency nextHop : nextHops) {
 728             String rd = primaryRd;
 729             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 730             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 731                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 732                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 733                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 734                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 735                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 736                 continue;
 737             }
 738             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 739                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 740                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 741                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 742                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 743                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 744                 writeOperTxn.merge(
 745                     LogicalDatastoreType.OPERATIONAL,
 746                     VpnUtil.getPrefixToInterfaceIdentifier(
 747                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 748                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 749                             prefixCue), true);
 750                 final Uuid subnetId = nextHop.getSubnetId();
 751
 752                 String gatewayIp = nextHop.getSubnetGatewayIp();
 753                 if (gatewayIp == null) {
 754                     Optional<String> gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 755                     if (gatewayIpOptional.isPresent()) {
 756                         gatewayIp = gatewayIpOptional.get();
 757                     }
 758                 }
 759
 760                 if (gatewayIp != null) {
 761                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 762                     if (gwMac.isPresent()) {
 763                         // A valid mac-address is available for this subnet-gateway-ip
 764                         // Use this for programming ARP_RESPONDER table here.  And save this
 765                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 766                         // to populate L3_GW_MAC_TABLE there.
 767                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 768                                 gatewayIp, gwMac.get());
 769                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 770                     } else {
 771                         // A valid mac-address is not available for this subnet-gateway-ip
 772                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 773                         // Save this connected-mac-address as gateway-mac-address for the
 774                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 775                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 776                         if (gwMac.isPresent()) {
 777                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 778                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 779                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 780                                     gatewayIp, gwMac.get());
 781                         } else {
 782                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet could not be "
 783                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 784                                 + "gwIp {}",
 785                                 interfaceName, vpnName, gatewayIp);
 786                         }
 787                     }
 788                 } else {
 789                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 790                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 791                         subnetId, interfaceName, vpnName);
 792                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 793                 }
 794                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 795                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 796             } else {
 797                 //Extra route adjacency
 798                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 799                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 800                 synchronized (vpnPrefixKey.intern()) {
 801                     java.util.Optional<String> rdToAllocate = VpnUtil
 802                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 803                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 804                     if (rdToAllocate.isPresent()) {
 805                         rd = rdToAllocate.get();
 806                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 807                             rd, prefix);
 808                     } else {
 809                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 810                         continue;
 811                     }
 812                 }
 813                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 814                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 815                         interfaceName, dpnId);
 816             }
 817             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 818             // can be different from the gateway-mac-address within the VRFEntry as the
 819             // gateway-mac-address is a superset.
 820             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 821                     : RouteOrigin.STATIC;
 822             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 823                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 824                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 825             Adjacency operationalAdjacency = null;
 826             try {
 827                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 828             } catch (NullPointerException e) {
 829                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 830                     input, e.getMessage());
 831                 return;
 832             }
 833             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 834                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 835                         vpnName, l3vni, origin,
 836                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 837             }
 838             value.add(operationalAdjacency);
 839         }
 840
 841         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 842         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 843                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 844
 845         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 846                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 847                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 848
 849         for (Adjacency nextHop : aug.getAdjacency()) {
 850             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 851             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 852                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 853                         : RouteOrigin.STATIC;
 854                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 855                 registeredPopulator.populateFib(input, writeConfigTxn);
 856             }
 857         }
 858     }
 859
 860     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 861                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 862         VpnInterfaceOpDataEntry opInterface =
 863               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 864         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 865             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 866         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 867                 WriteTransaction.CREATE_MISSING_PARENTS);
 868         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 869                 interfaceName, dpnId, vpnName);
 870     }
 871
 872     // TODO Clean up the exception handling
 873     @SuppressWarnings("checkstyle:IllegalCatch")
 874     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 875                                            StateTunnelList stateTunnelList,
 876                                            WriteTransaction writeConfigTxn,
 877                                            WriteTransaction writeOperTxn) {
 878
 879         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 880         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 881         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 882         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 883         if (adjList.isEmpty()) {
 884             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 885                     vpnInterface, srcDpnId);
 886             return;
 887         }
 888         String prefix = null;
 889         long label = 0;
 890         List<Adjacency> value = new ArrayList<>();
 891         boolean isNextHopAddReqd = false;
 892         String vpnName = vpnInterface.getVpnInstanceName();
 893         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 894         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 895         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 896                 vpnInterface.getName(), vpnInterface.getDpnId(),
 897                 vpnInterface.getVpnInstanceName(), adjList);
 898         for (Adjacency adj : adjList) {
 899             String rd = adj.getVrfId();
 900             rd = rd != null ? rd : vpnName;
 901             prefix = adj.getIpAddress();
 902             label = adj.getLabel();
 903             List<String> nhList = Collections.singletonList(srcTepIp);
 904             List<String> nextHopList = adj.getNextHopIpList();
 905             // If TEP is added , update the nexthop of primary adjacency.
 906             // Secondary adj nexthop is already pointing to primary adj IP address.
 907             if (nextHopList != null && !nextHopList.isEmpty() && nextHopList.get(0).equalsIgnoreCase(srcTepIp)) {
 908                 /* everything right already */
 909             } else {
 910                 isNextHopAddReqd = true;
 911             }
 912
 913             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 914                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 915             } else {
 916                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 917                 if (!vrfEntryOptional.isPresent()) {
 918                     continue;
 919                 }
 920                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 921                 if (!nhList.contains(srcTepIp)) {
 922                     nhList.add(srcTepIp);
 923                     isNextHopAddReqd = true;
 924                 }
 925                 value.add(adj);
 926             }
 927
 928             if (isNextHopAddReqd) {
 929                 updateLabelMapper(label, nhList);
 930                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 931                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 932                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 933                 // Update the VRF entry with nextHop
 934                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 935                         label, true, writeConfigTxn);
 936
 937                 //Get the list of VPN's importing this route(prefix) .
 938                 // Then update the VRF entry with nhList
 939                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 940                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 941                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 942                     String vpnRd = vpn.getVrfId();
 943                     if (vpnRd != null) {
 944                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 945                                 srcTepIp, label, true, writeConfigTxn);
 946                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 947                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 948                             vpnInterface.getName(), srcDpnId, vpnName,
 949                             vpn.getVpnInstanceName(), vpnRd);
 950                     }
 951                 }
 952                 // Advertise the prefix to BGP only for external vpn
 953                 // since there is a nexthop change.
 954                 try {
 955                     if (!rd.equalsIgnoreCase(vpnName)) {
 956                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 957                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 958                                 null /*gatewayMacAddress*/);
 959                     }
 960                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 961                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 962                             srcDpnId, vpnName);
 963                 } catch (Exception ex) {
 964                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 965                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 966                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 967                 }
 968             }
 969         }
 970         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 971         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 972                 .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 973                 .addAugmentation(AdjacenciesOp.class, aug).build();
 974         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 975                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 976         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 977                 WriteTransaction.CREATE_MISSING_PARENTS);
 978         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 979                 vpnInterface.getName(), srcDpnId, vpnName);
 980
 981     }
 982
 983     // TODO Clean up the exception handling
 984     @SuppressWarnings("checkstyle:IllegalCatch")
 985     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
 986                                               StateTunnelList stateTunnelList,
 987                                               WriteTransaction writeConfigTxn,
 988                                               WriteTransaction writeOperTxn) {
 989
 990         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 991         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 992         String prefix = null;
 993         long label = 0;
 994         boolean isNextHopRemoveReqd = false;
 995         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 996         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 997         String vpnName = vpnInterface.getVpnInstanceName();
 998         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 999         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1000         if (adjList != null) {
1001             List<Adjacency> value = new ArrayList<>();
1002             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1003                     vpnInterface.getName(), vpnInterface.getDpnId(),
1004                     vpnInterface.getVpnInstanceName(), adjList);
1005             for (Adjacency adj : adjList) {
1006                 List<String> nhList = new ArrayList<>();
1007                 String rd = adj.getVrfId();
1008                 rd = rd != null ? rd : vpnName;
1009                 prefix = adj.getIpAddress();
1010                 List<String> nextHopList = adj.getNextHopIpList();
1011                 label = adj.getLabel();
1012                 if (nextHopList != null && !nextHopList.isEmpty()) {
1013                     isNextHopRemoveReqd = true;
1014                 }
1015                 // If TEP is deleted , remove the nexthop from primary adjacency.
1016                 // Secondary adj nexthop will continue to point to primary adj IP address.
1017                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1018                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1019                 } else {
1020                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1021                     if (!vrfEntryOptional.isPresent()) {
1022                         continue;
1023                     }
1024                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1025                     if (nhList.contains(srcTepIp)) {
1026                         nhList.remove(srcTepIp);
1027                         isNextHopRemoveReqd = true;
1028                     }
1029                     value.add(adj);
1030                 }
1031
1032                 if (isNextHopRemoveReqd) {
1033                     updateLabelMapper(label, nhList);
1034                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1035                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1036                             prefix, nhList, vpnName,
1037                             vpnId, rd, vpnInterface.getName());
1038                     // Update the VRF entry with removed nextHop
1039                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1040                             label, false, writeConfigTxn);
1041
1042                     //Get the list of VPN's importing this route(prefix) .
1043                     // Then update the VRF entry with nhList
1044                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1045                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1046                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1047                         String vpnRd = vpn.getVrfId();
1048                         if (vpnRd != null) {
1049                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1050                                     srcTepIp, label, false, writeConfigTxn);
1051                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1052                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1053                                     nhList, label, vpnInterface.getName(), srcDpnId,
1054                                     vpnName,
1055                                     vpn.getVpnInstanceName(), vpnRd);
1056                         }
1057                     }
1058
1059                     // Withdraw prefix from BGP only for external vpn.
1060                     try {
1061                         if (!rd.equalsIgnoreCase(vpnName)) {
1062                             bgpManager.withdrawPrefix(rd, prefix);
1063                         }
1064                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1065                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1066                                 vpnInterface.getName(), srcDpnId,
1067                                 vpnName);
1068                     } catch (Exception ex) {
1069                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1070                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1071                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1072                     }
1073                 }
1074             }
1075             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1076             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1077                     .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1078                     .addAugmentation(AdjacenciesOp.class, aug).build();
1079             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1080                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1081             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1082                     WriteTransaction.CREATE_MISSING_PARENTS);
1083             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1084                          vpnInterface.getName(), srcDpnId, vpnName);
1085         }
1086     }
1087
1088     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1089         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1090
1091         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1092         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1093         if (vpnInstanceOpDataEntry == null) {
1094             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1095                     + " to check for vpns exporting the routes", vpnName);
1096             return vpnsToExportRoute;
1097         }
1098
1099         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1100             if (input.getVpnInstanceName() == null) {
1101                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1102                         input.getVrfId());
1103                 return false;
1104             }
1105             return !input.getVpnInstanceName().equals(vpnName);
1106         };
1107
1108         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1109             Iterable<String> commonRTs =
1110                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1111                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1112             return Iterators.size(commonRTs.iterator()) > 0;
1113         };
1114
1115         vpnsToExportRoute =
1116                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1117                         Collectors.toList());
1118         return vpnsToExportRoute;
1119     }
1120
1121     // TODO Clean up the exception handling
1122     @SuppressWarnings("checkstyle:IllegalCatch")
1123     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1124         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1125         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1126             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1127             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1128             if (vrfEntries != null) {
1129                 for (VrfEntry vrfEntry : vrfEntries) {
1130                     try {
1131                         if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1132                                 RouteOrigin.value(vrfEntry.getOrigin()))) {
1133                             LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1134                                     + " is not a controller managed non intervpn link route. Ignoring.",
1135                                     vpn.getVrfId(), vrfEntry.getDestPrefix());
1136                             continue;
1137                         }
1138                         String prefix = vrfEntry.getDestPrefix();
1139                         String gwMac = vrfEntry.getGatewayMacAddress();
1140                         vrfEntry.getRoutePaths().forEach(routePath -> {
1141                             String nh = routePath.getNexthopAddress();
1142                             int label = routePath.getLabel().intValue();
1143                             if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1144                                     vrfEntry.getOrigin()))) {
1145                                 LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1146                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1147                                         vpnName, vpnRd);
1148                                 fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1149                                         Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1150                                         0 /*l3vni*/, gwMac,  null /*parentVpnRd*/, RouteOrigin.SELF_IMPORTED,
1151                                         writeConfigTxn);
1152                             } else {
1153                                 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} prefix {}"
1154                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1155                                         vpnName, vpnRd);
1156                                 SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1157                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, writeConfigTxn);
1158                             }
1159                         });
1160                     } catch (RuntimeException e) {
1161                         LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1162                                 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1163                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1164                     }
1165                 }
1166                 writeConfigTxn.submit();
1167             } else {
1168                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1169                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1170             }
1171         }
1172     }
1173
1174     @SuppressWarnings("checkstyle:IllegalCatch")
1175     @Override
1176     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1177         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1178         final String interfaceName = key.getName();
1179         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1180             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1181             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1182         }
1183     }
1184
1185     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1186                                 final VpnInterface vpnInterface, final String vpnName,
1187                                 final String interfaceName) {
1188         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1189             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1190                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1191                 deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn, vpnName);
1192                 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1193                 ListenableFuture<Void> futures = writeConfigTxn.submit();
1194                 String errorText = "removeVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
1195                         + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
1196                 ListenableFutures.addErrorLogging(futures, LOG, errorText);
1197                 return Collections.singletonList(futures);
1198             }, DJC_MAX_RETRIES);
1199         } else {
1200             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1201             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1202         }
1203     }
1204
1205     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1206     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1207                                 final VpnInterface vpnInterface, final String vpnName,
1208                                 final String interfaceName, final Interface interfaceState) {
1209         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1210                 vpnName, vpnInterface.getDpnId());
1211         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1212         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1213             () -> {
1214                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1215                 ListenableFuture<Void> configFuture = txRunner
1216                         .callWithNewWriteOnlyTransactionAndSubmit(writeConfigTxn -> {
1217                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeOperTxn -> {
1218                                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1219                                     LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1220                                             interfaceName, vpnName);
1221                                     BigInteger dpId = BigInteger.ZERO;
1222                                     int ifIndex = 0;
1223                                     String gwMacAddress = null;
1224                                     InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1225                                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1226                                     final Optional<VpnInterfaceOpDataEntry> optVpnInterface =
1227                                             VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1228                                     if (interfaceState != null) {
1229                                         try {
1230                                             dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1231                                         } catch (NumberFormatException | IllegalStateException e) {
1232                                             LOG.error("remove: Unable to retrieve dpnId from interface operational"
1233                                                     + " data store for interface {} on dpn {} for vpn {} Fetching"
1234                                                             + " from vpn interface op data store. ", interfaceName,
1235                                                     vpnInterface.getDpnId(), vpnName, e);
1236                                             dpId = BigInteger.ZERO;
1237                                         }
1238                                         ifIndex = interfaceState.getIfIndex();
1239                                         gwMacAddress = interfaceState.getPhysAddress().getValue();
1240                                     } else {
1241                                         LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1242                                                 + " from vpn interface op.", interfaceName);
1243                                         if (optVpnInterface.isPresent()) {
1244                                             VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1245                                             dpId = vpnOpInterface.getDpnId();
1246                                             ifIndex = vpnOpInterface.getLportTag().intValue();
1247                                             gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1248                                         } else {
1249                                             LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1250                                                             + " as interfaceState and vpn interface op is not"
1251                                                     + " available", interfaceName, vpnName);
1252                                             return;
1253                                         }
1254                                     }
1255                                     processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1256                                             optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1257                                             writeConfigTxn, writeOperTxn, writeInvTxn);
1258                                     LOG.info(
1259                                             "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1260                                                     + "successfully",
1261                                             interfaceName, vpnInterface.getDpnId(), vpnName);
1262                                 }));
1263                             }));
1264                         });
1265                 futures.add(configFuture);
1266                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1267                 return futures;
1268             }, DJC_MAX_RETRIES);
1269     }
1270
1271     protected void processVpnInterfaceDown(BigInteger dpId,
1272                                            String interfaceName,
1273                                            int lportTag,
1274                                            String gwMac,
1275                                            VpnInterfaceOpDataEntry vpnOpInterface,
1276                                            boolean isInterfaceStateDown,
1277                                            WriteTransaction writeConfigTxn,
1278                                            WriteTransaction writeOperTxn,
1279                                            WriteTransaction writeInvTxn) {
1280         if (vpnOpInterface == null) {
1281             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1282                     + " as it is not available in operational data store", interfaceName, dpId);
1283             return;
1284         }
1285         final String vpnName = vpnOpInterface.getVpnInstanceName();
1286         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1287                                                     interfaceName, vpnName);
1288         if (!isInterfaceStateDown) {
1289             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1290             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1291                     null);
1292             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1293             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1294                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1295             if (interfaceManager.isExternalInterface(interfaceName)) {
1296                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1297                         NwConstants.DEL_FLOW);
1298             }
1299             if (!isBgpVpnInternetVpn) {
1300                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1301             }
1302             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1303                     + " successful", interfaceName, dpId, vpnName);
1304         } else {
1305             // Interface is retained in the DPN, but its Link Down.
1306             // Only withdraw the prefixes for this interface from BGP
1307             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1308         }
1309     }
1310
1311     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1312                                           final String vpnName, final long vpnId, String gwMac,
1313                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1314                                           final WriteTransaction writeInvTxn) {
1315         //Read NextHops
1316         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1317                 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1318         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1319         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1320
1321         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1322         LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1323                 interfaceName, dpnId, vpnName, primaryRd);
1324         if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1325             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1326             LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1327                     interfaceName, dpnId, vpnName, nextHops);
1328             for (Adjacency nextHop : nextHops) {
1329                 if (nextHop.isPhysNetworkFunc()) {
1330                     LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1331                             nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1332                     fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1333                             null/*writeCfgTxn*/);
1334                 } else {
1335                     String rd = nextHop.getVrfId();
1336                     List<String> nhList;
1337                     if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1338                         nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1339                     } else {
1340                         // This is a primary adjacency
1341                         nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1342                                 : Collections.emptyList();
1343                         removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1344                                 interfaceName, writeInvTxn);
1345                     }
1346                     if (!nhList.isEmpty()) {
1347                         if (rd.equals(vpnName)) {
1348                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1349                             //remove from FIB directly
1350                             nhList.forEach((nh) -> removeAdjacencyFromInternalVpn(nextHop, vpnName,
1351                                     interfaceName, dpnId, writeConfigTxn));
1352                         } else {
1353                             removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, interfaceName,
1354                                     writeConfigTxn);
1355                         }
1356                     } else {
1357                         LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1358                                         + " interface {}", nextHop.getIpAddress(), rd,
1359                                 nextHop.getAdjacencyType().toString(), interfaceName);
1360                         bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1361                         fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1362                     }
1363                 }
1364                 String ip = nextHop.getIpAddress().split("/")[0];
1365                 LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1366                 if (vpnVipToPort != null) {
1367                     VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip);
1368                     LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1369                                     + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(),
1370                             ip, dpnId, vpnName);
1371                 }
1372             }
1373         } else {
1374             // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1375             LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1376                     + " Removing it.", interfaceName, vpnName, dpnId);
1377             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1378         }
1379     }
1380
1381     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1382                                                                    String interfaceName, BigInteger dpnId,
1383                                                                    WriteTransaction writeConfigTxn) {
1384         return (nh) -> {
1385             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1386                     writeConfigTxn);
1387             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1388                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1389                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1390         };
1391     }
1392
1393     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1394                                            BigInteger dpnId, String rd, String interfaceName,
1395                                            WriteTransaction writeConfigTxn) {
1396         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1397                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1398         nhList.forEach((nh) -> {
1399             //IRT: remove routes from other vpns importing it
1400             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1401                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1402             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1403                 String vpnRd = vpn.getVrfId();
1404                 if (vpnRd != null) {
1405                     fibManager.removeOrUpdateFibEntry(vpnRd,
1406                             nextHop.getIpAddress(), nh, writeConfigTxn);
1407                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1408                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1409                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1410                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1411                 }
1412             }
1413         });
1414     }
1415
1416     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1417                                                  int lportTag, String gwMac, String interfaceName,
1418                                                  WriteTransaction writeInvTxn) {
1419         final Uuid subnetId = nextHop.getSubnetId();
1420         if (nextHop.getSubnetGatewayMacAddress() == null) {
1421             // A valid mac-address was not available for this subnet-gateway-ip
1422             // So a connected-mac-address was used for this subnet and we need
1423             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1424             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1425                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1426         }
1427         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1428                 subnetId);
1429     }
1430
1431     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1432                                                   String interfaceName) {
1433         // This is either an extra-route (or) a learned IP via subnet-route
1434         List<String> nhList = null;
1435         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1436         if (nextHopIp == null || nextHopIp.isEmpty()) {
1437             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1438                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1439                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1440                     vpnName);
1441             nhList = Collections.emptyList();
1442         } else {
1443             nhList = Collections.singletonList(nextHopIp);
1444         }
1445         return nhList;
1446     }
1447
1448     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1449         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1450         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1451         // else use connected interface
1452         if (gwPort != null && gwPort.isSubnetIp()) {
1453             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1454                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1455             return Optional.of(gwPort.getMacAddress());
1456         }
1457         return Optional.absent();
1458     }
1459
1460     @Override
1461     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1462         final VpnInterface update) {
1463         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1464                 update.getDpnId(), original.getVpnInstanceNames(),
1465                 update.getVpnInstanceNames());
1466         final String vpnInterfaceName = update.getName();
1467         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1468         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1469         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1470             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1471         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1472         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1473             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1474
1475         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1476         //handles switching between <internal VPN - external VPN>
1477         if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1478             LOG.info("update: handled VPNInterface {} on dpn {} update"
1479                      + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1480                      original.getName(), dpnId,
1481                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1482                      VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1483             return;
1484         }
1485         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1486             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1487             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1488             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1489             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1490             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1491                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1492                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1493                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1494                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1495                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1496                     LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1497                             update.getName(), newVpnName);
1498                     //handle both addition and removal of adjacencies
1499                     //currently, new adjacency may be an extra route
1500                     boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1501                     if (!oldAdjs.equals(newAdjs)) {
1502                         for (Adjacency adj : copyNewAdjs) {
1503                             if (copyOldAdjs.contains(adj)) {
1504                                 copyOldAdjs.remove(adj);
1505                             } else {
1506                                 // add new adjacency - right now only extra route will hit this path
1507                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1508                                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1509                                              dpnId, writeOperTxn, writeConfigTxn);
1510                                 }
1511                                 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to vpn "
1512                                                 + "interface {} on vpn {} dpnId {}",
1513                                         adj.getIpAddress(), adj.getNextHopIpList(),
1514                                         adj.getLabel(), adj.getSubnetId(), update.getName(),
1515                                         newVpnName, dpnId);
1516                             }
1517                         }
1518                         for (Adjacency adj : copyOldAdjs) {
1519                             if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1520                                 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1521                                        writeOperTxn, writeConfigTxn);
1522                             }
1523                             LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1524                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1525                                 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1526                         }
1527                     }
1528                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
1529                     try {
1530                         operFuture.get();
1531                     } catch (ExecutionException e) {
1532                         LOG.error("Exception encountered while submitting operational future for update"
1533                                 + " VpnInterface {} on vpn {}", vpnInterfaceName, newVpnName, e);
1534                         return null;
1535                     }
1536                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1537                     futures.add(writeConfigTxn.submit());
1538                     LOG.info("update: vpn interface updated for interface {} oldVpn(s) {} newVpn {}"
1539                         + "processed successfully", update.getName(),
1540                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), newVpnName);
1541                     return futures;
1542                 });
1543             } else {
1544                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1545                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1546             }
1547         }
1548     }
1549
1550     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1551                                                  VpnInterface original, VpnInterface update) {
1552         boolean isSwap = Boolean.FALSE;
1553         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1554         final String interfaceName = key.getName();
1555         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1556             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1557         List<String> oldVpnListCopy = new ArrayList<>();
1558         oldVpnListCopy.addAll(oldVpnList);
1559         List<String> newVpnList = update.getVpnInstanceNames().stream()
1560             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1561         oldVpnList.removeAll(newVpnList);
1562         for (String oldVpnName: oldVpnList) {
1563             isSwap = Boolean.TRUE;
1564             LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {} running"
1565                      + " config-driven swap removal", interfaceName, oldVpnName);
1566             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1567             LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1568                      + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1569         }
1570         //Wait for previous interface bindings to be removed
1571         try {
1572             Thread.sleep(2000);
1573         } catch (InterruptedException e) {
1574             //Ignore
1575         }
1576         newVpnList.removeAll(oldVpnListCopy);
1577         for (String newVpnName: newVpnList) {
1578             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1579             isSwap = Boolean.TRUE;
1580             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1581                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1582                          + "running config-driven swap addition", interfaceName, newVpnName);
1583                 final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1584                 final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1585                                                 ? origAdjs.getAdjacency() : new ArrayList<>();
1586                 final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1587                 final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1588                                                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1589
1590                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1591                 LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1592                          + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1593                          interfaceName, oldVpnListCopy, newVpnName);
1594             }
1595         }
1596         return isSwap;
1597     }
1598
1599     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1600         Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1601         synchronized (label.toString().intern()) {
1602             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1603                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1604             Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1605             if (opResult.isPresent()) {
1606                 LabelRouteInfo labelRouteInfo =
1607                     new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1608                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1609             }
1610         }
1611         LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList);
1612     }
1613
1614     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1615         SubnetRoute route, WriteTransaction writeConfigTxn) {
1616
1617         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1618         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, null /* parentVpnRd */)
1619                 .addAugmentation(SubnetRoute.class, route).build();
1620         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1621         InstanceIdentifierBuilder<VrfTables> idBuilder =
1622             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1623         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1624         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1625         if (writeConfigTxn != null) {
1626             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1627         } else {
1628             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1629         }
1630         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1631                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1632     }
1633
1634     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1635                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1636                                            WriteTransaction writeConfigTxn) {
1637
1638         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1639                                                 LogicalDatastoreType.OPERATIONAL, identifier);
1640
1641         if (optVpnInterface.isPresent()) {
1642             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1643             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1644             String vpnName = currVpnIntf.getVpnInstanceName();
1645             VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1646             InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1647             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker,
1648                                             LogicalDatastoreType.OPERATIONAL, adjPath);
1649             boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1650             VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1651             long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1652             VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1653             List<Adjacency> adjacencies;
1654             if (optAdjacencies.isPresent()) {
1655                 adjacencies = optAdjacencies.get().getAdjacency();
1656             } else {
1657                 // This code will be hit in case of first PNF adjacency
1658                 adjacencies = new ArrayList<>();
1659             }
1660             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1661             L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1662                     .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1663             Adjacency operationalAdjacency = null;
1664             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) {
1665                 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1666                         : RouteOrigin.STATIC;
1667                 String nh = adj.getNextHopIpList().get(0);
1668                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1669                 synchronized (vpnPrefixKey.intern()) {
1670                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1671                             dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1672                     if (rdToAllocate.isPresent()) {
1673                         input.setRd(rdToAllocate.get());
1674                         operationalAdjacency = populator.createOperationalAdjacency(input);
1675                         int label = operationalAdjacency.getLabel().intValue();
1676                         vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1677                                 currVpnIntf.getVpnInstanceName(), l3vni, origin,
1678                                 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1679                         LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1680                                 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1681                                 vpnName, label, currVpnIntf.getName(), dpnId);
1682                     } else {
1683                         LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName,
1684                                 prefix);
1685                         return;
1686                     }
1687                     // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1688                     // Keeping the MPLS check for now.
1689                     if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1690                         final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1691                         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1692                                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1693                         vpnsToImportRoute.forEach(vpn -> {
1694                             if (vpn.getVrfId() != null) {
1695                                 VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1696                                         dataBroker, vpn.getVpnId(), vpnId, prefix,
1697                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1698                                         .ifPresent(
1699                                             rds -> vpnManager.addExtraRoute(
1700                                                     VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1701                                                     adj.getIpAddress(), nh, rds,
1702                                                     currVpnIntf.getVpnInstanceName(),
1703                                                     l3vni, RouteOrigin.SELF_IMPORTED,
1704                                                     currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn));
1705                             }
1706                         });
1707                     }
1708                 }
1709             } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1710                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1711                         currVpnIntf.getName(), vpnName);
1712
1713                 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1714
1715                 writeOperTxn.merge(
1716                         LogicalDatastoreType.OPERATIONAL,
1717                         VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1718                                 adj.getSubnetId().getValue()), prefix),
1719                         VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1720                                 adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1721
1722                 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1723                         adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, 0 /*l3vni*/,
1724                       null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1725
1726                 input.setRd(adj.getVrfId());
1727             }
1728             if (operationalAdjacency == null) {
1729                 operationalAdjacency = populator.createOperationalAdjacency(input);
1730             }
1731             adjacencies.add(operationalAdjacency);
1732             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1733             VpnInterfaceOpDataEntry newVpnIntf =
1734                 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1735                     aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1736                         currVpnIntf.getGatewayMacAddress());
1737
1738             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1739         }
1740     }
1741
1742     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1743         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1744         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1745     }
1746
1747     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1748             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1749         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1750                                           LogicalDatastoreType.OPERATIONAL, identifier);
1751
1752         if (optVpnInterface.isPresent()) {
1753             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1754
1755             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1756             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1757             if (optAdjacencies.isPresent()) {
1758                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1759
1760                 if (!adjacencies.isEmpty()) {
1761                     LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1762                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1763                     while (adjIt.hasNext()) {
1764                         Adjacency adjElem = adjIt.next();
1765                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1766                             String rd = adjElem.getVrfId();
1767                             adjIt.remove();
1768
1769                             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1770                             VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1771                                     .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1772                                     currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove(),
1773                                             currVpnIntf.getLportTag(), currVpnIntf.getGatewayMacAddress());
1774
1775                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1776                             if (adj.getNextHopIpList() != null) {
1777                                 for (String nh : adj.getNextHopIpList()) {
1778                                     deleteExtraRouteFromCurrentAndImportingVpns(
1779                                         currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1780                                         currVpnIntf.getName(), writeConfigTxn);
1781                                 }
1782                             } else if (adj.isPhysNetworkFunc()) {
1783                                 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1784                                         adj.getIpAddress(), adj.getSubnetId());
1785                                 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1786                                         writeConfigTxn);
1787                             }
1788                             break;
1789                         }
1790
1791                     }
1792                 }
1793                 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1794                         dpnId, adj.getVrfId());
1795             } else {
1796                 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1797                         + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1798             }
1799         }
1800     }
1801
1802     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1803         String rd, String intfName, WriteTransaction writeConfigTxn) {
1804         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1805         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1806         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1807             String vpnRd = vpn.getVrfId();
1808             if (vpnRd != null) {
1809                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1810             }
1811         }
1812     }
1813
1814     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1815         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1816             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1817             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1818     }
1819
1820     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1821         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1822             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1823     }
1824
1825     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1826         WriteTransaction writeOperTxn) {
1827         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1828         if (dpId.equals(BigInteger.ZERO)) {
1829             LOG.error("addToNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1830                     + " association model", vpnInterfaceName, routerName);
1831
1832             return;
1833         }
1834         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1835
1836         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1837             .OPERATIONAL, routerDpnListIdentifier);
1838         RouterInterfaces routerInterface =
1839             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1840                 vpnInterfaceName).build();
1841         if (optionalRouterDpnList.isPresent()) {
1842             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1843                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1844         } else {
1845             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1846             builder.setRouterId(routerName);
1847             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1848             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
1849             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1850                 getRouterId(routerName),
1851                 builder.build(), true);
1852         }
1853     }
1854
1855     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1856         WriteTransaction writeOperTxn) {
1857         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1858         if (dpId.equals(BigInteger.ZERO)) {
1859             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1860                     + " dissociation model", vpnInterfaceName, routerName);
1861
1862             return;
1863         }
1864         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1865         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1866             .OPERATIONAL, routerDpnListIdentifier);
1867         if (optionalRouterDpnList.isPresent()) {
1868             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1869             RouterInterfaces routerInterface =
1870                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1871                     vpnInterfaceName).build();
1872
1873             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1874                 if (routerInterfaces.isEmpty()) {
1875                     if (writeOperTxn != null) {
1876                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1877                     } else {
1878                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1879                     }
1880                 } else {
1881                     if (writeOperTxn != null) {
1882                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1883                             RouterInterfaces.class,
1884                             new RouterInterfacesKey(vpnInterfaceName)));
1885                     } else {
1886                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1887                             routerDpnListIdentifier.child(
1888                                 RouterInterfaces.class,
1889                                 new RouterInterfacesKey(vpnInterfaceName)));
1890                     }
1891                 }
1892             }
1893         }
1894     }
1895
1896     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1897         WriteTransaction writeOperTxn) {
1898         if (dpId.equals(BigInteger.ZERO)) {
1899             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1900                     + " dissociation model", vpnInterfaceName, routerName);
1901
1902             return;
1903         }
1904         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1905         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1906             .OPERATIONAL, routerDpnListIdentifier);
1907         if (optionalRouterDpnList.isPresent()) {
1908             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1909             RouterInterfaces routerInterface =
1910                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1911                     vpnInterfaceName).build();
1912             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1913                 if (routerInterfaces.isEmpty()) {
1914                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1915                 } else {
1916                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1917                         RouterInterfaces.class,
1918                         new RouterInterfacesKey(vpnInterfaceName)));
1919                 }
1920             }
1921         }
1922     }
1923
1924     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1925                                                     WriteTransaction writeConfigTxn, String vpnName) {
1926         if (vpnInterface == null) {
1927             return;
1928         }
1929         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1930         if (adjs == null) {
1931             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1932                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1933             return;
1934         }
1935         for (Adjacency adj : adjs) {
1936             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1937                 String primaryInterfaceIp = adj.getIpAddress();
1938                 String macAddress = adj.getMacAddress();
1939                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1940
1941                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1942                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1943
1944                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1945                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1946                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1947                         routerInt, label, writeConfigTxn);
1948                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1949                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1950                         macAddress);
1951                 return;
1952             }
1953         }
1954         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1955                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1956                 primaryRd, vpnName);
1957     }
1958
1959     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1960                                                     WriteTransaction writeConfigTxn, String vpnName) {
1961         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
1962         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1963         if (adjs != null) {
1964             List<Adjacency> adjsList = adjs.getAdjacency();
1965             for (Adjacency adj : adjsList) {
1966                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1967                     String primaryInterfaceIp = adj.getIpAddress();
1968                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1969                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1970                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1971                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1972                     return;
1973                 }
1974             }
1975         } else {
1976             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1977                     vpnInterface.getName(), rd);
1978         }
1979     }
1980
1981     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1982         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1983             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1984                   vpnName, intefaceData.vpnInterface.getName());
1985             return;
1986         }
1987         final VpnInterfaceKey key = intefaceData.identifier
1988                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1989         final String interfaceName = key.getName();
1990         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1991                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1992         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1993                   intefaceData.identifier, vpnName);
1994         return;
1995     }
1996
1997     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1998                                               VpnInterface vpnInterface, String vpnName) {
1999         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2000                .get(vpnName);
2001         if (vpnInterfaces == null) {
2002             vpnInterfaces = new ConcurrentLinkedQueue<>();
2003         }
2004         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2005         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2006         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2007                 vpnInterface.getName(), vpnName);
2008     }
2009
2010     public boolean isVpnInstanceReady(String vpnInstanceName) {
2011         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2012         if (vpnRd == null) {
2013             return false;
2014         }
2015         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2016
2017         return vpnInstanceOpDataEntry != null;
2018     }
2019
2020     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2021         synchronized (vpnInstanceName.intern()) {
2022             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2023                     unprocessedVpnInterfaces.get(vpnInstanceName);
2024             if (vpnInterfaces != null) {
2025                 while (!vpnInterfaces.isEmpty()) {
2026                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2027                     if (hasVpnInstanceCreatedSuccessfully) {
2028                         processSavedInterface(savedInterface, vpnInstanceName);
2029                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2030                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2031                     } else {
2032                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2033                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2034                     }
2035                 }
2036             } else {
2037                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2038             }
2039         }
2040     }
2041
2042     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2043             VpnInterface vpnInterface) {
2044         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2045             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2046                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2047             if (vpnInterfaces != null) {
2048                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2049                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2050                             + "unprocessed list", vpnInterface.getName(),
2051                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2052                 }
2053             } else {
2054                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2055                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2056             }
2057         }
2058     }
2059
2060     public void vpnInstanceIsReady(String vpnInstanceName) {
2061         processSavedInterfaces(vpnInstanceName, true);
2062     }
2063
2064     public void vpnInstanceFailed(String vpnInstanceName) {
2065         processSavedInterfaces(vpnInstanceName, false);
2066     }
2067
2068     private static class UnprocessedVpnInterfaceData {
2069         InstanceIdentifier<VpnInterface> identifier;
2070         VpnInterface vpnInterface;
2071
2072         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2073             this.identifier = identifier;
2074             this.vpnInterface = vpnInterface;
2075         }
2076
2077         @Override
2078         public int hashCode() {
2079             final int prime = 31;
2080             int result = 1;
2081             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2082             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2083             return result;
2084         }
2085
2086         @Override
2087         public boolean equals(Object obj) {
2088             if (this == obj) {
2089                 return true;
2090             }
2091             if (obj == null) {
2092                 return false;
2093             }
2094             if (getClass() != obj.getClass()) {
2095                 return false;
2096             }
2097             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2098             if (identifier == null) {
2099                 if (other.identifier != null) {
2100                     return false;
2101                 }
2102             } else if (!identifier.equals(other.identifier)) {
2103                 return false;
2104             }
2105             if (vpnInterface == null) {
2106                 if (other.vpnInterface != null) {
2107                     return false;
2108                 }
2109             } else if (!vpnInterface.equals(other.vpnInterface)) {
2110                 return false;
2111             }
2112             return true;
2113         }
2114     }
2115
2116     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2117         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2118         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2119         if (vpnInstanceOpData == null) {
2120             return;
2121         }
2122         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2123         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2124             return;
2125         }
2126         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2127         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2128             InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2129                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2130             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = VpnUtil.read(dataBroker,
2131                     LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2132             if (!vpnInterfaceOptional.isPresent()) {
2133                 return;
2134             }
2135             List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2136                     vpnInterface.getInterfaceName());
2137             if (configVpnAdjacencies == null) {
2138                 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2139                 return;
2140             }
2141             List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2142                     .getAugmentation(AdjacenciesOp.class).getAdjacency();
2143             // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2144             // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2145             // in operational DS. These unprocessed adjacencies will be handled below.
2146             // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2147             // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2148             configVpnAdjacencies.stream()
2149                     .filter(adjacency -> operationVpnAdjacencies.stream()
2150                             .noneMatch(operationalAdjacency ->
2151                                     operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2152                     .forEach(adjacency -> {
2153                         LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2154                         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2155                             () -> {
2156                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
2157                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
2158                                 if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2159                                     addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd, adjacency,
2160                                             vpnInterfaceOptional.get().getDpnId(), writeConfigTxn, writeOperTxn);
2161                                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
2162                                     try {
2163                                         operFuture.get();
2164                                     } catch (ExecutionException | InterruptedException e) {
2165                                         LOG.error("Exception encountered while submitting operational"
2166                                                 + " future for vpnInterface {}", vpnInterface, e);
2167                                     }
2168                                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2169                                     futures.add(writeConfigTxn.submit());
2170                                     return futures;
2171                                 }  else {
2172                                     return Collections.emptyList();
2173                                 }
2174                             });
2175                     });
2176         }));
2177     }
2178
2179     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2180         private final String interfaceName;
2181         private final boolean add;
2182         private final String txnDestination;
2183
2184         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2185             this.interfaceName = interfaceName;
2186             this.add = add;
2187             this.txnDestination = transactionDest;
2188         }
2189
2190         @Override
2191         public void onSuccess(Void voidObj) {
2192             if (add) {
2193                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2194                         interfaceName, txnDestination);
2195             } else {
2196                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2197             }
2198         }
2199
2200         @Override
2201         public void onFailure(Throwable throwable) {
2202             if (add) {
2203                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2204                         interfaceName, txnDestination, throwable);
2205             } else {
2206                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2207                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
2208             }
2209         }
2210     }
2211 }