vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.ListenableFuture;
  14
  15 import java.math.BigInteger;
  16 import java.util.ArrayList;
  17 import java.util.Collection;
  18 import java.util.Collections;
  19 import java.util.HashSet;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.Set;
  25 import java.util.TimerTask;
  26 import java.util.concurrent.BlockingQueue;
  27 import java.util.concurrent.ConcurrentHashMap;
  28 import java.util.concurrent.ConcurrentLinkedQueue;
  29 import java.util.concurrent.ExecutionException;
  30 import java.util.concurrent.Executors;
  31 import java.util.concurrent.LinkedBlockingQueue;
  32 import java.util.concurrent.ScheduledThreadPoolExecutor;
  33 import java.util.concurrent.TimeUnit;
  34 import java.util.function.Predicate;
  35 import java.util.stream.Collectors;
  36
  37 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  38 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  41 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  42 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  43 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  47 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  48 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  49 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  50 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  51 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  52 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  53 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  54 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  55 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  56 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  57 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  58 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  59 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnTargets;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 108 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 109 import org.opendaylight.yangtools.yang.common.RpcError;
 110 import org.slf4j.Logger;
 111 import org.slf4j.LoggerFactory;
 112
 113 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager>
 114     implements AutoCloseable {
 115
 116     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 117     private static final int VPN_INF_UPDATE_TIMER_TASK_DELAY = 1000;
 118     private static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
 119
 120     private final DataBroker dataBroker;
 121     private final IBgpManager bgpManager;
 122     private final IFibManager fibManager;
 123     private final IMdsalApiManager mdsalManager;
 124     private final IdManagerService idManager;
 125     private final OdlArputilService arpManager;
 126     private final OdlInterfaceRpcService ifaceMgrRpcService;
 127     private final VpnFootprintService vpnFootprintService;
 128     private final IInterfaceManager interfaceManager;
 129     private final IVpnManager vpnManager;
 130     private final IVpnLinkService ivpnLinkService;
 131     private final ArpResponderHandler arpResponderHandler;
 132
 133     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 134
 135     private BlockingQueue<UpdateData> vpnInterfacesUpdateQueue = new LinkedBlockingQueue<>();
 136     private ScheduledThreadPoolExecutor vpnInfUpdateTaskExecutor = (ScheduledThreadPoolExecutor) Executors
 137             .newScheduledThreadPool(1);
 138
 139     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 140             new ConcurrentHashMap<>();
 141
 142     public VpnInterfaceManager(final DataBroker dataBroker,
 143                                final IBgpManager bgpManager,
 144                                final OdlArputilService arpManager,
 145                                final IdManagerService idManager,
 146                                final IMdsalApiManager mdsalManager,
 147                                final IFibManager fibManager,
 148                                final OdlInterfaceRpcService ifaceMgrRpcService,
 149                                final VpnFootprintService vpnFootprintService,
 150                                final IInterfaceManager interfaceManager,
 151                                final IVpnManager vpnManager,
 152                                final IVpnLinkService ivpnLnkSrvce,
 153                                final ArpResponderHandler arpResponderHandler) {
 154         super(VpnInterface.class, VpnInterfaceManager.class);
 155
 156         this.dataBroker = dataBroker;
 157         this.bgpManager = bgpManager;
 158         this.arpManager = arpManager;
 159         this.idManager = idManager;
 160         this.mdsalManager = mdsalManager;
 161         this.fibManager = fibManager;
 162         this.ifaceMgrRpcService = ifaceMgrRpcService;
 163         this.vpnFootprintService = vpnFootprintService;
 164         this.interfaceManager = interfaceManager;
 165         this.vpnManager = vpnManager;
 166         this.ivpnLinkService = ivpnLnkSrvce;
 167         this.arpResponderHandler = arpResponderHandler;
 168         vpnInfUpdateTaskExecutor.scheduleWithFixedDelay(new VpnInterfaceUpdateTimerTask(),
 169             0, VPN_INF_UPDATE_TIMER_TASK_DELAY, TIME_UNIT);
 170     }
 171
 172     public Runnable isNotifyTaskQueued(String intfName) {
 173         return vpnIntfMap.remove(intfName);
 174     }
 175
 176     public void start() {
 177         LOG.info("{} start", getClass().getSimpleName());
 178         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 179     }
 180
 181     @Override
 182     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 183         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 184     }
 185
 186     @Override
 187     protected VpnInterfaceManager getDataTreeChangeListener() {
 188         return VpnInterfaceManager.this;
 189     }
 190
 191     private InstanceIdentifier<Interface> getInterfaceListenerPath() {
 192         return InstanceIdentifier.create(InterfacesState.class).child(Interface.class);
 193     }
 194
 195     @Override
 196     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 197         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(), vpnInterface.getVpnInstanceName());
 198         if (canHandleNewVpnInterface(identifier, vpnInterface)) {
 199             addVpnInterface(identifier, vpnInterface, null, null);
 200         } else {
 201             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 202                     vpnInterface.getVpnInstanceName(), vpnInterface.getName());
 203         }
 204     }
 205
 206     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 207             final VpnInterface vpnInterface) {
 208         synchronized (vpnInterface.getVpnInstanceName().intern()) {
 209             if (isVpnInstanceReady(vpnInterface.getVpnInstanceName())) {
 210                 return true;
 211             }
 212             addToUnprocessedVpnInterfaces(identifier, vpnInterface);
 213             return false;
 214         }
 215     }
 216
 217     // TODO Clean up the exception handling
 218     @SuppressWarnings("checkstyle:IllegalCatch")
 219     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 220         final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 221         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 222         final String interfaceName = key.getName();
 223         final String vpnName = vpnInterface.getVpnInstanceName();
 224
 225         LOG.info("addVpnInterface: VPN Interface add event - intfName {} onto vpnName {} from addVpnInterface",
 226                 interfaceName, vpnName);
 227         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 228         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 229             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 230             if (interfaceState != null) {
 231                 try {
 232                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 233                     final int ifIndex = interfaceState.getIfIndex();
 234                     DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 235                     dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 236                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 237                         WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 238                         WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 239                         LOG.info("addVpnInterface: VPN Interface add event - intfName {} vpnName {} on dpn {}" ,
 240                                 vpnInterface.getName(), vpnInterface.getVpnInstanceName(), vpnInterface.getDpnId());
 241                         processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, writeConfigTxn,
 242                                 writeOperTxn, writeInvTxn, interfaceState);
 243                         if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 244                             LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 245                                     + " Update for swapping VPN {} case.", interfaceName,
 246                                     vpnInterface.getVpnInstanceName());
 247                             if (newAdjs != null) {
 248                                 for (Adjacency adj : newAdjs) {
 249                                     if (oldAdjs.contains(adj)) {
 250                                         oldAdjs.remove(adj);
 251                                     } else {
 252                                         addNewAdjToVpnInterface(identifier, primaryRd, adj, dpnId, writeOperTxn,
 253                                                 writeConfigTxn);
 254                                     }
 255                                 }
 256                             }
 257                             for (Adjacency adj : oldAdjs) {
 258                                 delAdjFromVpnInterface(identifier, adj, dpnId, writeOperTxn, writeConfigTxn);
 259                             }
 260                         }
 261                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
 262                         try {
 263                             operFuture.get();
 264                         } catch (ExecutionException e) {
 265                             LOG.error("addVpnInterface: Exception encountered while submitting operational future for"
 266                                     + " addVpnInterface {} on vpn {}: {}", vpnInterface.getName(), vpnName, e);
 267                             return null;
 268                         }
 269                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 270                         futures.add(writeConfigTxn.submit());
 271                         futures.add(writeInvTxn.submit());
 272                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 273                                 + " processed successfully", interfaceName, vpnInterface.getVpnInstanceName(), dpnId);
 274                         return futures;
 275                     });
 276                 } catch (Exception e) {
 277                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 278                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 279                             vpnInterface.getVpnInstanceName(), e);
 280                     return;
 281                 }
 282             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 283                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 284                 dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 285                     () -> {
 286                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 287                         createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName, writeConfigTxn);
 288                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 289                         futures.add(writeConfigTxn.submit());
 290                         LOG.info("addVpnInterface: Router Interface {} for vpn {} on dpn {} added successfully.",
 291                                 interfaceName, vpnInterface.getVpnInstanceName(), vpnInterface.getDpnId());
 292                         return futures;
 293                     });
 294             } else {
 295                 LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 296                         + " is not available", interfaceName, vpnName);
 297             }
 298         } else {
 299             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 300                     + " as interfaceState is not available", interfaceName, vpnInterface.getVpnInstanceName(),
 301                     vpnInterface.getDpnId());
 302         }
 303     }
 304
 305     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 306             final int lportTag, boolean isInterfaceUp,
 307             WriteTransaction writeConfigTxn,
 308             WriteTransaction writeOperTxn,
 309             WriteTransaction writeInvTxn,
 310             Interface interfaceState) {
 311
 312         final String interfaceName = vpnInterface.getName();
 313         if (!isInterfaceUp) {
 314             final String vpnName = vpnInterface.getVpnInstanceName();
 315             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 316                     interfaceName, dpId, vpnName);
 317             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 318             if (vpnId == VpnConstants.INVALID_ID) {
 319                 LOG.error("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 320                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 321                         dpId);
 322                 return;
 323             }
 324
 325             boolean waitForVpnInterfaceOpRemoval = false;
 326             VpnInterface opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, vpnInterface.getName());
 327             if (opVpnInterface != null) {
 328                 String opVpnName = opVpnInterface.getVpnInstanceName();
 329                 String primaryInterfaceIp = null;
 330                 if (opVpnName.equals(vpnName)) {
 331                     // Please check if the primary VRF Entry does not exist for VPNInterface
 332                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 333                     // back to back
 334                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 335                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 336                     if (adjs == null) {
 337                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 338                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 339                                 vpnName);
 340                         return;
 341                     }
 342                     for (Adjacency adj : adjs) {
 343                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 344                             primaryInterfaceIp = adj.getIpAddress();
 345                             break;
 346                         }
 347                     }
 348                     if (primaryInterfaceIp == null) {
 349                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 350                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 351                                 dpId, vpnName);
 352                         return;
 353                     }
 354                     // Get the rd of the vpn instance
 355                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 356                     if (vrf != null) {
 357                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 358                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 359                         return;
 360                     }
 361                     waitForVpnInterfaceOpRemoval = true;
 362                 } else {
 363                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 364                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 365                 }
 366             }
 367             if (!waitForVpnInterfaceOpRemoval) {
 368                 // Add the VPNInterface and quit
 369                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 370                         null/*ipAddressSourceValuePair*/,
 371                         true /* add */);
 372                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/);
 373                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 374                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 375                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 376                         dpId, vpnName);
 377                 if (interfaceManager.isExternalInterface(interfaceName)) {
 378                     processExternalVpnInterface(vpnInterface, vpnId, dpId, lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 379                 }
 380                 return;
 381             }
 382
 383             // FIB didn't get a chance yet to clean up this VPNInterface
 384             // Let us give it a chance here !
 385             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 386                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 387             try {
 388                 Runnable notifyTask = new VpnNotifyTask();
 389                 vpnIntfMap.put(interfaceName, notifyTask);
 390                 synchronized (notifyTask) {
 391                     try {
 392                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 393                     } catch (InterruptedException e) {
 394                         // Ignored
 395                     }
 396                 }
 397             } finally {
 398                 vpnIntfMap.remove(interfaceName);
 399             }
 400
 401             opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
 402             if (opVpnInterface != null) {
 403                 LOG.error("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 404                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 405                         dpId, vpnName);
 406                 return;
 407             }
 408             // VPNInterface got removed, proceed with Add
 409             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 410                     interfaceName, dpId, vpnName);
 411             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 412                     null/*ipAddressSourceValuePair*/,
 413                     true /* add */);
 414             VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/);
 415             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 416                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 417             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 418                     + " FIB to clean up", interfaceName, dpId, vpnName);
 419             if (interfaceManager.isExternalInterface(interfaceName)) {
 420                 processExternalVpnInterface(vpnInterface, vpnId, dpId, lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 421             }
 422         } else {
 423             // Interface is retained in the DPN, but its Link Up.
 424             // Advertise prefixes again for this interface to BGP
 425             InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 426             advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, identifier, vpnInterface);
 427
 428             // Perform similar operation as interface add event for extraroutes.
 429             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 430             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 431             if (!optAdjacencies.isPresent()) {
 432                 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 433                 return;
 434             }
 435             List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 436             for (Adjacency adjacency : adjacencies) {
 437                 if (adjacency.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 438                     addNewAdjToVpnInterface(identifier, primaryRd, adjacency, dpId, writeOperTxn, writeConfigTxn);
 439                 }
 440             }
 441         }
 442     }
 443
 444     private void processExternalVpnInterface(VpnInterface vpnInterface, long vpnId, BigInteger dpId, int lportTag,
 445             WriteTransaction writeInvTxn, int addOrRemove) {
 446         Uuid extNetworkId;
 447         try {
 448             // vpn instance of ext-net interface is the network-id
 449             extNetworkId = new Uuid(vpnInterface.getVpnInstanceName());
 450         } catch (IllegalArgumentException e) {
 451             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 452                     + " on dpn {} failed", vpnInterface.getVpnInstanceName(), vpnInterface.getName(), dpId);
 453             return;
 454         }
 455
 456         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 457         if (routerIds == null || routerIds.isEmpty()) {
 458             LOG.error("processExternalVpnInterface: No router is associated with {}."
 459                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 460                     extNetworkId.getValue(), vpnInterface.getName(), dpId,
 461                     vpnInterface.getVpnInstanceName());
 462             return;
 463         }
 464
 465         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 466                 + " for vpn {}", routerIds, vpnInterface.getName(), dpId,
 467                 vpnInterface.getVpnInstanceName());
 468         for (Uuid routerId : routerIds) {
 469             String routerName = routerId.getValue();
 470             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 471             if (Objects.equals(primarySwitch, dpId)) {
 472                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 473                 if (router != null) {
 474                     vpnManager.setupArpResponderFlowsToExternalNetworkIps(routerName,
 475                             VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 476                             dpId, vpnId, vpnInterface.getName(), lportTag, writeInvTxn, addOrRemove);
 477                 } else {
 478                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 479                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 480                             vpnInterface.getName(), dpId, vpnInterface.getVpnInstanceName());
 481                 }
 482             }
 483         }
 484     }
 485
 486     // TODO Clean up the exception handling
 487     @SuppressWarnings("checkstyle:IllegalCatch")
 488     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 489                                                  final InstanceIdentifier<VpnInterface> identifier,
 490                                                  VpnInterface intf) {
 491         if (rd == null) {
 492             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 493                     intf.getName(), dpnId, intf.getVpnInstanceName());
 494             return;
 495         } else {
 496             if (rd.equals(intf.getVpnInstanceName())) {
 497                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 498                         + " as it is in internal vpn{} with rd {}", intf.getName(), dpnId,
 499                         intf.getVpnInstanceName(), rd);
 500
 501                 return;
 502             }
 503         }
 504         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 505                 intf.getName(), dpnId, intf.getVpnInstanceName(), rd);
 506
 507         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 508         if (nextHopIp == null) {
 509             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 510                     + " returning from advertising route with rd {} vpn {} to bgp", intf.getName(), dpnId,
 511                     rd, intf.getVpnInstanceName());
 512             return;
 513         }
 514
 515         //Read NextHops
 516         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 517         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 518         if (adjacencies.isPresent()) {
 519             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 520
 521             if (!nextHops.isEmpty()) {
 522                 LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 523                         + " rd {}", nextHops, intf.getName(), dpnId, intf.getVpnInstanceName(), rd);
 524                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 525                 long l3vni = vpnInstanceOpData.getL3vni();
 526                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 527                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 528                 for (Adjacency nextHop : nextHops) {
 529                     if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 530                         continue;
 531                     }
 532                     String gatewayMac = null;
 533                     long label = 0;
 534                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 535                         final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 536                                 vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 537                         gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, intf.getName()).get();
 538                     } else {
 539                         label = nextHop.getLabel();
 540                     }
 541                     try {
 542                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 543                                 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 544                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 545                                 encapType, (int)label, l3vni, 0 /*l2vni*/,
 546                                 gatewayMac);
 547                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 548                                 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 549                                 nextHop.getIpAddress(), nextHopIp, label, intf.getName(), dpnId,
 550                                 intf.getVpnInstanceName());
 551                     } catch (Exception e) {
 552                         LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}"
 553                                 + " for interface {} on dpn {}", nextHop.getIpAddress(), intf.getVpnInstanceName(), rd,
 554                                 intf.getName(), dpnId, e);
 555                     }
 556                 }
 557             }
 558         }
 559     }
 560
 561     // TODO Clean up the exception handling
 562     @SuppressWarnings("checkstyle:IllegalCatch")
 563     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier,
 564             VpnInterface intf, WriteTransaction writeConfigTxn) {
 565         //Read NextHops
 566         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 567         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 568
 569         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 570         if (rd == null) {
 571             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 572                     intf.getName(), intf.getVpnInstanceName());
 573             return;
 574         } else {
 575             if (rd.equals(intf.getVpnInstanceName())) {
 576                 LOG.info(
 577                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 578                                 + "internal vpn{} with rd {}",
 579                         intf.getName(), intf.getVpnInstanceName(), rd);
 580                 return;
 581             }
 582         }
 583         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 584                 intf.getVpnInstanceName(), rd);
 585         if (adjacencies.isPresent()) {
 586             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 587
 588             if (!nextHops.isEmpty()) {
 589                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 590                         nextHops, intf.getName(), intf.getVpnInstanceName(), rd);
 591                 for (Adjacency nextHop : nextHops) {
 592                     try {
 593                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 594                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 595                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 596                                     intf.getName(), intf.getVpnInstanceName());
 597                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 598                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 599                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 600                                     intf.getName(), intf.getVpnInstanceName());
 601                         } else {
 602                             // Perform similar operation as interface delete event for extraroutes.
 603                             String allocatedRd = nextHop.getVrfId();
 604                             for (String nh : nextHop.getNextHopIpList()) {
 605                                 deleteExtraRouteFromCurrentAndImportingVpns(intf.getVpnInstanceName(),
 606                                         nextHop.getIpAddress(), nh, allocatedRd, intf.getName(), writeConfigTxn);
 607                             }
 608                         }
 609                     } catch (Exception e) {
 610                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 611                                 + " for interface {} ", nextHop.getIpAddress(), intf.getVpnInstanceName(), rd,
 612                                 intf.getName(), e);
 613                     }
 614                 }
 615             }
 616         }
 617     }
 618
 619     @SuppressWarnings("checkstyle:IllegalCatch")
 620     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 621                                                   String primaryRd, String interfaceName, final long vpnId,
 622                                                   WriteTransaction writeConfigTxn,
 623                                                   WriteTransaction writeOperTxn,
 624                                                   final WriteTransaction writeInvTxn,
 625                                                   Interface interfaceState) {
 626         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 627         // Read NextHops
 628         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 629         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 630         if (!adjacencies.isPresent()) {
 631             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null, writeOperTxn);
 632             return;
 633         }
 634
 635         // Get the rd of the vpn instance
 636         String nextHopIp = null;
 637         try {
 638             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 639         } catch (Exception e) {
 640             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve enpoint ip address for "
 641                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 642         }
 643         List<String> nhList = new ArrayList<>();
 644         if (nextHopIp != null) {
 645             nhList.add(nextHopIp);
 646             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 647                     interfaceName, dpnId, vpnName, nhList);
 648         }
 649         Optional<String> gwMac = Optional.absent();
 650         String vpnInterfaceSubnetGwMacAddress = null;
 651         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 652         Long l3vni = vpnInstanceOpData.getL3vni();
 653         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 654         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 655         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 656         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 657         List<Adjacency> value = new ArrayList<>();
 658         for (Adjacency nextHop : nextHops) {
 659             String rd = primaryRd;
 660             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 661                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 662                 boolean isNatPrefix = nextHop.isPhysNetworkFunc() ? true : false;
 663                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 664                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 665                 writeOperTxn.merge(
 666                     LogicalDatastoreType.OPERATIONAL,
 667                     VpnUtil.getPrefixToInterfaceIdentifier(
 668                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 669                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 670                             isNatPrefix), true);
 671                 final Uuid subnetId = nextHop.getSubnetId();
 672                 final Optional<String> gatewayIp = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 673                 if (gatewayIp.isPresent()) {
 674                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp.get());
 675                     if (gwMac.isPresent()) {
 676                         // A valid mac-address is available for this subnet-gateway-ip
 677                         // Use this for programming ARP_RESPONDER table here.  And save this
 678                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 679                         // to populate L3_GW_MAC_TABLE there.
 680                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, vpnName, vpnId, interfaceName,
 681                                 subnetId, gatewayIp.get(), gwMac.get());
 682                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 683                     } else {
 684                         // A valid mac-address is not available for this subnet-gateway-ip
 685                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 686                         // Save this connected-mac-address as gateway-mac-address for the
 687                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 688                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 689                         if (gwMac.isPresent()) {
 690                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 691                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, interfaceState);
 692                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, vpnName, vpnId, interfaceName,
 693                                     subnetId, gatewayIp.get(), gwMac.get());
 694                         } else {
 695                             LOG.error("Gateway MAC for subnet ID {} could not be obtained, cannot create "
 696                                             + "ARP responder flow for interface name {}, vpnName {}, gwIp {}",
 697                                     interfaceName, vpnName, gatewayIp.get());
 698                         }
 699                     }
 700                 } else {
 701                     LOG.warn("Gateway IP for subnet ID {} could not be obtained, cannot create ARP responder flow "
 702                             + "for interface name {}, vpnName {}", subnetId, interfaceName, vpnName);
 703                 }
 704                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 705                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 706             } else {
 707                 //Extra route adjacency
 708                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 709                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 710                 synchronized (vpnPrefixKey.intern()) {
 711                     java.util.Optional<String> rdToAllocate = VpnUtil
 712                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 713                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId, writeOperTxn);
 714                     if (rdToAllocate.isPresent()) {
 715                         rd = rdToAllocate.get();
 716                         LOG.info("The rd {} is allocated for the extraroute {}", rd, prefix);
 717                     } else {
 718                         LOG.error("No rds to allocate extraroute {}", prefix);
 719                         continue;
 720                     }
 721                 }
 722                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 723                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 724                         interfaceName, dpnId);
 725             }
 726             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 727             // can be different from the gateway-mac-address within the VRFEntry as the
 728             // gateway-mac-address is a superset.
 729             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 730                     : RouteOrigin.STATIC;
 731             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 732                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 733                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress)
 734                 .setRouteOrigin(origin);
 735             Adjacency operationalAdjacency = null;
 736             try {
 737                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 738             } catch (NullPointerException e) {
 739                 LOG.error(e.getMessage());
 740                 return;
 741             }
 742             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 743                 addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 744                         vpnName, operationalAdjacency.getLabel().intValue(), l3vni, origin,
 745                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 746             }
 747             value.add(operationalAdjacency);
 748         }
 749
 750         Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 751         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, writeOperTxn);
 752
 753         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 754                 .setGatewayMac(gwMac.isPresent() ? gwMac.get() : null).setInterfaceName(interfaceName)
 755                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 756         for (Adjacency nextHop : aug.getAdjacency()) {
 757             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 758             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 759                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 760                         : RouteOrigin.STATIC;
 761                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 762                 registeredPopulator.populateFib(input, writeConfigTxn, writeOperTxn);
 763             }
 764         }
 765     }
 766
 767     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, Adjacencies aug,
 768         WriteTransaction writeOperTxn) {
 769         VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 770         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 771         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 772                 WriteTransaction.CREATE_MISSING_PARENTS);
 773         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 774                 interfaceName, dpnId, opInterface.getVpnInstanceName());
 775     }
 776
 777     // TODO Clean up the exception handling
 778     @SuppressWarnings("checkstyle:IllegalCatch")
 779     public void updateVpnInterfaceOnTepAdd(VpnInterface vpnInterface,
 780                                            StateTunnelList stateTunnelList,
 781                                            WriteTransaction writeConfigTxn,
 782                                            WriteTransaction writeOperTxn) {
 783
 784         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 785         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 786         Adjacencies adjacencies = vpnInterface.getAugmentation(Adjacencies.class);
 787         List<Adjacency> adjList = (adjacencies != null) ? adjacencies.getAdjacency() : new ArrayList<>();
 788         if (adjList.isEmpty()) {
 789             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 790                     vpnInterface, srcDpnId);
 791             return;
 792         }
 793         String prefix = null;
 794         long label = 0;
 795         List<String> nhList = new ArrayList<>();
 796         boolean isNextHopAddReqd = false;
 797         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInterface.getVpnInstanceName());
 798         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnInterface.getVpnInstanceName());
 799         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 800                 vpnInterface.getName(), vpnInterface.getDpnId(), vpnInterface.getVpnInstanceName(),
 801                 adjList);
 802         for (Adjacency adj : adjList) {
 803             String rd = adj.getVrfId();
 804             rd = (rd != null) ? rd : vpnInterface.getVpnInstanceName();
 805             prefix = adj.getIpAddress();
 806             label = adj.getLabel();
 807             nhList = Collections.singletonList(srcTepIp);
 808             List<String> nextHopList = adj.getNextHopIpList();
 809             // If TEP is added , update the nexthop of primary adjacency.
 810             // Secondary adj nexthop is already pointing to primary adj IP address.
 811             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 812                 if (!(nextHopList != null && (!nextHopList.isEmpty())
 813                         && (nextHopList.get(0).equalsIgnoreCase(srcTepIp)))) {
 814                     isNextHopAddReqd = true;
 815                     LOG.trace("updateVpnInterfaceOnTepAdd: NextHopList to be updated {} for vpnInterface {} on dpn {} "
 816                             + "and adjacency {}", nhList, vpnInterface, srcDpnId, adj);
 817                     InstanceIdentifier<Adjacency> adjId =
 818                             VpnUtil.getAdjacencyIdentifier(vpnInterface.getName(), prefix);
 819                     MDSALUtil.syncWrite(dataBroker,  LogicalDatastoreType.OPERATIONAL, adjId,
 820                             new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 821                 }
 822             } else {
 823                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 824                 if (!vrfEntryOptional.isPresent()) {
 825                     continue;
 826                 }
 827                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 828                 if (!nhList.contains(srcTepIp)) {
 829                     nhList.add(srcTepIp);
 830                     isNextHopAddReqd = true;
 831                 }
 832             }
 833
 834             if (isNextHopAddReqd) {
 835                 updateLabelMapper(label, nhList);
 836                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 837                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 838                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 839                 // Update the VRF entry with nextHop
 840                 fibManager.updateRoutePathForFibEntry(dataBroker, primaryRd, prefix, srcTepIp, label, true, null);
 841
 842                 //Get the list of VPN's importing this route(prefix) .
 843                 // Then update the VRF entry with nhList
 844                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 845                         getVpnsImportingMyRoute(vpnInterface.getVpnInstanceName());
 846                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 847                     String vpnRd = vpn.getVrfId();
 848                     if (vpnRd != null) {
 849                         fibManager.updateRoutePathForFibEntry(dataBroker, vpnRd, prefix,
 850                                 srcTepIp, label, true, null);
 851                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 852                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 853                                 vpnInterface.getName(), srcDpnId, vpnInterface.getVpnInstanceName(),
 854                                 vpn.getVpnInstanceName(), vpnRd);
 855                     }
 856                 }
 857                 // Advertise the prefix to BGP only for external vpn
 858                 // since there is a nexthop change.
 859                 try {
 860                     if (!rd.equalsIgnoreCase(vpnInterface.getVpnInstanceName())) {
 861                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 862                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 863                                 null /*gatewayMacAddress*/);
 864                     }
 865                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 866                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 867                             srcDpnId, vpnInterface.getVpnInstanceName());
 868                 } catch (Exception ex) {
 869                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 870                             + " on rd {} for interface {} on dpn {} vpn {} as {}", prefix, nhList, label, rd,
 871                             vpnInterface.getName(), srcDpnId, vpnInterface.getVpnInstanceName(), ex);
 872                 }
 873             }
 874             LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 875                     vpnInterface.getName(), srcDpnId, vpnInterface.getVpnInstanceName());
 876         }
 877     }
 878
 879     // TODO Clean up the exception handling
 880     @SuppressWarnings("checkstyle:IllegalCatch")
 881     public void updateVpnInterfaceOnTepDelete(VpnInterface vpnInterface,
 882                                               StateTunnelList stateTunnelList,
 883                                               WriteTransaction writeConfigTxn,
 884                                               WriteTransaction writeOperTxn) {
 885
 886         Adjacencies adjacencies = vpnInterface.getAugmentation(Adjacencies.class);
 887         List<Adjacency> adjList = (adjacencies != null) ? adjacencies.getAdjacency() : new ArrayList<>();
 888         String prefix = null;
 889         long label = 0;
 890         boolean isNextHopRemoveReqd = false;
 891         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 892         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 893         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInterface.getVpnInstanceName());
 894         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnInterface.getVpnInstanceName());
 895
 896         if (adjList != null) {
 897             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 898                     vpnInterface.getName(), vpnInterface.getDpnId(), vpnInterface.getVpnInstanceName(), adjList);
 899             for (Adjacency adj : adjList) {
 900                 List<String> nhList = new ArrayList<>();
 901                 String rd = adj.getVrfId();
 902                 rd = (rd != null) ? rd : vpnInterface.getVpnInstanceName();
 903                 prefix = adj.getIpAddress();
 904                 label = adj.getLabel();
 905                 // If TEP is deleted , remove the nexthop from primary adjacency.
 906                 // Secondary adj nexthop will continue to point to primary adj IP address.
 907                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 908                     List<String> nextHopList = adj.getNextHopIpList();
 909                     if ((nextHopList != null) & (!nextHopList.isEmpty())) {
 910                         isNextHopRemoveReqd = true;
 911                         InstanceIdentifier<Adjacency> adjId =
 912                                 VpnUtil.getAdjacencyIdentifier(vpnInterface.getName(), prefix);
 913                         MDSALUtil.syncWrite(dataBroker,  LogicalDatastoreType.OPERATIONAL, adjId,
 914                                 new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 915                     }
 916                 } else {
 917                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 918                     if (!vrfEntryOptional.isPresent()) {
 919                         continue;
 920                     }
 921                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 922                     if (nhList.contains(srcTepIp)) {
 923                         nhList.remove(srcTepIp);
 924                         isNextHopRemoveReqd = true;
 925                     }
 926                 }
 927
 928                 if (isNextHopRemoveReqd) {
 929                     updateLabelMapper(label, nhList);
 930                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
 931                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
 932                             prefix, nhList, vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 933                     // Update the VRF entry with removed nextHop
 934                     fibManager.updateRoutePathForFibEntry(dataBroker, primaryRd, prefix, srcTepIp, label, false, null);
 935
 936                     //Get the list of VPN's importing this route(prefix) .
 937                     // Then update the VRF entry with nhList
 938                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 939                         getVpnsImportingMyRoute(vpnInterface.getVpnInstanceName());
 940                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 941                         String vpnRd = vpn.getVrfId();
 942                         if (vpnRd != null) {
 943                             fibManager.updateRoutePathForFibEntry(dataBroker, vpnRd, prefix,
 944                                     srcTepIp, label, false, null);
 945                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
 946                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
 947                                     nhList, label, vpnInterface.getName(), srcDpnId,
 948                                     vpnInterface.getVpnInstanceName(), vpn.getVpnInstanceName(), vpnRd);
 949                         }
 950                     }
 951
 952                     // Withdraw prefix from BGP only for external vpn.
 953                     try {
 954                         if (!rd.equalsIgnoreCase(vpnInterface.getVpnInstanceName())) {
 955                             bgpManager.withdrawPrefix(rd, prefix);
 956                         }
 957                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
 958                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
 959                                 vpnInterface.getName(), srcDpnId, vpnInterface.getVpnInstanceName());
 960                     } catch (Exception ex) {
 961                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
 962                                 + " on rd {} for interface {} on dpn {} vpn {} as {}", prefix, nhList, label, rd,
 963                                 vpnInterface.getName(), srcDpnId, vpnInterface.getVpnInstanceName(), ex);
 964                     }
 965                 }
 966             }
 967             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
 968                     vpnInterface.getName(), srcDpnId, vpnInterface.getVpnInstanceName());
 969         }
 970     }
 971
 972     //TODO (KIRAN) : Move to L3vpnPopulator.
 973     public List<VpnInstanceOpDataEntry> getVpnsImportingMyRoute(final String vpnName) {
 974         List<VpnInstanceOpDataEntry> vpnsToImportRoute = new ArrayList<>();
 975
 976         final String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 977         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
 978         if (vpnInstanceOpDataEntry == null) {
 979             LOG.error("getVpnsImportingMyRoute: Could not retrieve vpn instance op data for {}"
 980                     + " to check for vpns importing the routes", vpnName);
 981             return vpnsToImportRoute;
 982         }
 983
 984         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
 985             if (input.getVpnInstanceName() == null) {
 986                 LOG.error("getVpnsImportingMyRoute.excludeVpn: Received vpn instance with rd {} without a name.",
 987                         input.getVrfId());
 988                 return false;
 989             }
 990             return !input.getVpnInstanceName().equals(vpnName);
 991         };
 992
 993         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
 994             Iterable<String> commonRTs =
 995                 intersection(getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ExportExtcommunity),
 996                     getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 997             return Iterators.size(commonRTs.iterator()) > 0;
 998         };
 999
1000         vpnsToImportRoute = VpnUtil.getAllVpnInstanceOpData(dataBroker)
1001                 .stream()
1002                 .filter(excludeVpn)
1003                 .filter(matchRTs)
1004                 .collect(Collectors.toList());
1005         return vpnsToImportRoute;
1006     }
1007
1008     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1009         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1010
1011         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1012         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1013         if (vpnInstanceOpDataEntry == null) {
1014             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1015                     + " to check for vpns exporting the routes", vpnName);
1016             return vpnsToExportRoute;
1017         }
1018
1019         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1020             if (input.getVpnInstanceName() == null) {
1021                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1022                         input.getVrfId());
1023                 return false;
1024             }
1025             return !input.getVpnInstanceName().equals(vpnName);
1026         };
1027
1028         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1029             Iterable<String> commonRTs =
1030                 intersection(getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1031                     getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1032             return Iterators.size(commonRTs.iterator()) > 0;
1033         };
1034
1035         vpnsToExportRoute =
1036                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1037                         Collectors.toList());
1038         return vpnsToExportRoute;
1039     }
1040
1041     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
1042         Set<T> intersection = new HashSet<>(collection1);
1043         intersection.retainAll(collection2);
1044         return intersection;
1045     }
1046
1047     private List<String> getRts(VpnInstanceOpDataEntry vpnInstance, VpnTarget.VrfRTType rtType) {
1048         String name = vpnInstance.getVpnInstanceName();
1049         List<String> rts = new ArrayList<>();
1050         VpnTargets targets = vpnInstance.getVpnTargets();
1051         if (targets == null) {
1052             LOG.info("getRts: vpn targets not available for {}", name);
1053             return rts;
1054         }
1055         List<VpnTarget> vpnTargets = targets.getVpnTarget();
1056         if (vpnTargets == null) {
1057             LOG.info("getRts: vpnTarget values not available for {}", name);
1058             return rts;
1059         }
1060         for (VpnTarget target : vpnTargets) {
1061             //TODO: Check for RT type is Both
1062             if (target.getVrfRTType().equals(rtType) || target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
1063                 String rtValue = target.getVrfRTValue();
1064                 rts.add(rtValue);
1065             }
1066         }
1067         return rts;
1068     }
1069
1070     // TODO Clean up the exception handling
1071     @SuppressWarnings("checkstyle:IllegalCatch")
1072     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1073         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1074         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1075             String rd = vpn.getVrfId();
1076             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1077             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1078             if (vrfEntries != null) {
1079                 for (VrfEntry vrfEntry : vrfEntries) {
1080                     try {
1081                         if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1082                                 RouteOrigin.value(vrfEntry.getOrigin()))) {
1083                             LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1084                                     + " is not a controller managed non intervpn link route. Ignoring.",
1085                                     vpn.getVrfId(), vrfEntry.getDestPrefix());
1086                             continue;
1087                         }
1088                         String prefix = vrfEntry.getDestPrefix();
1089                         String gwMac = vrfEntry.getGatewayMacAddress();
1090                         vrfEntry.getRoutePaths().forEach(routePath -> {
1091                             String nh = routePath.getNexthopAddress();
1092                             int label = routePath.getLabel().intValue();
1093                             if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1094                                     vrfEntry.getOrigin()))) {
1095                                 LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1096                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1097                                         vpnName, vpnRd);
1098                                 fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, null /*macAddress*/, prefix,
1099                                         Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1100                                         0 /*l3vni*/, gwMac,  null /*parentVpnRd*/, RouteOrigin.SELF_IMPORTED,
1101                                         writeConfigTxn);
1102                             } else {
1103                                 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} prefix {}"
1104                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1105                                         vpnName, vpnRd);
1106                                 SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1107                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, writeConfigTxn);
1108                             }
1109                         });
1110                     } catch (Exception e) {
1111                         LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1112                                 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1113                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1114                     }
1115                 }
1116                 writeConfigTxn.submit();
1117             } else {
1118                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1119                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1120             }
1121         }
1122     }
1123
1124     @SuppressWarnings("checkstyle:IllegalCatch")
1125     @Override
1126     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1127         LOG.info("remove: VPN Interface remove event - intfName {} vpn {}" ,vpnInterface.getName(),
1128                 vpnInterface.getVpnInstanceName());
1129         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1130         final String interfaceName = key.getName();
1131         BigInteger dpId = BigInteger.ZERO;
1132         final String vpnName = vpnInterface.getVpnInstanceName();
1133
1134         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1135                 vpnInterface.getVpnInstanceName(), vpnInterface.getDpnId());
1136         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1137         Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1138         if (interfaceState != null) {
1139             try {
1140                 dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1141             } catch (Exception e) {
1142                 LOG.error("remove: Unable to retrieve dpnId from interface operational data store for interface {}"
1143                         + " on dpn {} for vpn {} Fetching from vpn interface op data store. ", interfaceName,
1144                         vpnInterface.getDpnId(), vpnInterface.getVpnInstanceName(), e);
1145                 dpId = BigInteger.ZERO;
1146             }
1147
1148             final int ifIndex = interfaceState.getIfIndex();
1149             final BigInteger dpnId = dpId;
1150             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1151             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1152                 () -> {
1153                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1154                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1155                     WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
1156                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1157
1158                     LOG.info("remove: - intfName {} onto vpnName {} running config-driven", interfaceName, vpnName);
1159                     InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1160                     final Optional<VpnInterface> optVpnInterface =
1161                             VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1162                     if (optVpnInterface.isPresent()) {
1163                         VpnInterface vpnOpInterface = optVpnInterface.get();
1164                         BigInteger finalDpnId = dpnId.equals(BigInteger.ZERO) ? vpnOpInterface.getDpnId() : dpnId;
1165                         processVpnInterfaceDown(finalDpnId, interfaceName, ifIndex, interfaceState, vpnOpInterface,
1166                                 false, writeConfigTxn, writeOperTxn, writeInvTxn);
1167                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
1168                         try {
1169                             operFuture.get();
1170                         } catch (ExecutionException e) {
1171                             LOG.error("remove: Exception encountered while submitting operational future for remove "
1172                                     + "VpnInterface {} on dpn {}: {}", vpnInterface.getName(), dpnId, e);
1173                             return null;
1174                         }
1175                         futures.add(writeConfigTxn.submit());
1176                         futures.add(writeInvTxn.submit());
1177                         LOG.info("remove: Removal of vpn interface {} on dpn {} for vpn {} processed successfully",
1178                                 interfaceName, vpnInterface.getDpnId(), vpnInterface.getVpnInstanceName());
1179                     } else {
1180                         LOG.error("remove: VPN interface {} on dpn {} for vpn {} was unavailable in operational data "
1181                                 + "store to handle remove event", interfaceName, vpnInterface.getDpnId(),
1182                                 vpnInterface.getVpnInstanceName());
1183                     }
1184                     return futures;
1185                 });
1186
1187         } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1188             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1189             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
1190                 () -> {
1191                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1192                     deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn);
1193                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1194                     futures.add(writeConfigTxn.submit());
1195                     LOG.info("remove: Router interface {} for vpn {} removed successfully.", interfaceName,
1196                             vpnInterface.getVpnInstanceName());
1197                     return futures;
1198                 });
1199         } else {
1200             LOG.error("remove: Handling removal of VPN interface {} on dpn {} for vpn {} skipped as"
1201                     + " interfaceState is not available", interfaceName, vpnInterface.getDpnId(),
1202                     vpnInterface.getVpnInstanceName());
1203         }
1204     }
1205
1206     protected void processVpnInterfaceDown(BigInteger dpId,
1207                                            String interfaceName,
1208                                            int lportTag,
1209                                            Interface interfaceState,
1210                                            VpnInterface vpnOpInterface,
1211                                            boolean isInterfaceStateDown,
1212                                            WriteTransaction writeConfigTxn,
1213                                            WriteTransaction writeOperTxn,
1214                                            WriteTransaction writeInvTxn) {
1215         if (vpnOpInterface == null) {
1216             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1217                     + " as it is not available in operational data store", interfaceName, dpId);
1218             return;
1219         }
1220         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1221         if (!isInterfaceStateDown) {
1222             final String vpnName = vpnOpInterface.getVpnInstanceName();
1223             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1224             if (!vpnOpInterface.isScheduledForRemove()) {
1225                 VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1226                         null);
1227                 removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnOpInterface.getVpnInstanceName(),
1228                         vpnId, writeConfigTxn, writeInvTxn, interfaceState);
1229                 if (interfaceManager.isExternalInterface(interfaceName)) {
1230                     processExternalVpnInterface(vpnOpInterface, vpnId, dpId, lportTag, writeInvTxn,
1231                             NwConstants.DEL_FLOW);
1232                 }
1233                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown);
1234                 LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1235                         + " successful", interfaceName, dpId, vpnName);
1236             } else {
1237                 LOG.info("processVpnInterfaceDown: Unbinding vpn service for interface {} on dpn for vpn {}"
1238                         + " has already been scheduled by a different event ", interfaceName, dpId,
1239                         vpnName);
1240                 return;
1241             }
1242         } else {
1243             // Interface is retained in the DPN, but its Link Down.
1244             // Only withdraw the prefixes for this interface from BGP
1245             withdrawAdjacenciesForVpnFromBgp(identifier, vpnOpInterface, writeConfigTxn);
1246         }
1247     }
1248
1249     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1250                                           final String vpnName, final long vpnId, WriteTransaction writeConfigTxn,
1251                                           final WriteTransaction writeInvTxn, Interface interfaceState) {
1252         //Read NextHops
1253         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1254         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1255         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1256
1257         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1258         LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1259                 interfaceName, dpnId, vpnName, primaryRd);
1260         if (adjacencies.isPresent()) {
1261             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1262             if (!nextHops.isEmpty()) {
1263                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are ",
1264                         interfaceName, dpnId, vpnName, nextHops);
1265                 for (Adjacency nextHop : nextHops) {
1266                     String rd = nextHop.getVrfId();
1267                     List<String> nhList = new ArrayList<>();
1268                     if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1269                         // This is either an extra-route (or) a learned IP via subnet-route
1270                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1271                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1272                             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1273                                     + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1274                                     + " for vpn {}", rd, nextHop.getIpAddress(), interfaceName, dpnId,
1275                                     vpnName);
1276                         } else {
1277                             nhList = Collections.singletonList(nextHopIp);
1278                         }
1279                     } else {
1280                         // This is a primary adjacency
1281                         nhList = nextHop.getNextHopIpList();
1282                         final Uuid subnetId = nextHop.getSubnetId();
1283                         if (nextHop.getSubnetGatewayMacAddress() == null) {
1284                             // A valid mac-address was not available for this subnet-gateway-ip
1285                             // So a connected-mac-address was used for this subnet and we need
1286                             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1287                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1288                                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, interfaceState);
1289
1290                         }
1291                         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, vpnName, vpnId,
1292                                 subnetId);
1293                     }
1294
1295                     if (!nhList.isEmpty()) {
1296                         if (rd.equals(vpnName)) {
1297                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1298                             //remove from FIB directly
1299                             for (String nh : nhList) {
1300                                 fibManager.removeOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), nh,
1301                                     writeConfigTxn);
1302                                 LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1303                                         + " nexthop {} for interface {} on dpn {} for internal vpn {}", vpnName,
1304                                         nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1305                             }
1306                         } else {
1307                             List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1308                             for (String nh : nhList) {
1309                                 //IRT: remove routes from other vpns importing it
1310                                 removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1311                                         nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1312                                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1313                                     String vpnRd = vpn.getVrfId();
1314                                     if (vpnRd != null) {
1315                                         fibManager.removeOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), nh,
1316                                             writeConfigTxn);
1317                                         LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1318                                                 + " prefix {} nextHop {} from VPN {} parentVpn {} for interface {}"
1319                                                 + " on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1320                                                 vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1321                                     }
1322                                 }
1323                             }
1324                         }
1325                     } else {
1326                         primaryRd = nextHop.isPhysNetworkFunc() ? nextHop.getSubnetId().getValue() : primaryRd;
1327                         fibManager.removeFibEntry(dataBroker, primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1328                     }
1329
1330                     String ip = nextHop.getIpAddress().split("/")[0];
1331                     LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1332                     if (vpnVipToPort != null) {
1333                         VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip);
1334                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1335                                 + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(),
1336                                 ip, dpnId, vpnName);
1337                     }
1338                 }
1339             }
1340         }
1341     }
1342
1343     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1344         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1345         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1346         // else use connected interface
1347         if (gwPort != null && gwPort.isSubnetIp()) {
1348             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1349                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1350             return Optional.of(gwPort.getMacAddress());
1351         }
1352         return Optional.absent();
1353     }
1354
1355     // TODO Clean up the exception handling
1356     @SuppressWarnings("checkstyle:IllegalCatch")
1357     private void removePrefixFromBGP(String primaryRd, String rd, String vpnName, String prefix, String nextHop,
1358                                      String tunnelIp, BigInteger dpnId, WriteTransaction writeConfigTxn) {
1359         try {
1360             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}", rd, prefix,
1361                     nextHop);
1362             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1363             synchronized (vpnNamePrefixKey.intern()) {
1364                 Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
1365                         .getVpnExtraroutes(dataBroker, vpnName, rd, prefix);
1366                 if (optVpnExtraRoutes.isPresent()) {
1367                     List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
1368                     if (nhList != null && nhList.size() > 1) {
1369                         // If nhList is more than 1, just update vpntoextraroute and prefixtointerface DS
1370                         // For other cases, remove the corresponding tep ip from fibentry and withdraw prefix
1371                         nhList.remove(nextHop);
1372                         VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1373                                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd, prefix),
1374                                 VpnUtil.getVpnToExtraroute(prefix, nhList));
1375                         MDSALUtil.syncDelete(dataBroker,
1376                                 LogicalDatastoreType.CONFIGURATION, VpnExtraRouteHelper.getUsedRdsIdentifier(
1377                                 VpnUtil.getVpnId(dataBroker, vpnName), prefix, nextHop));
1378                         LOG.debug("removePrefixFromBGP: Removed vpn-to-extraroute with rd {} prefix {} nexthop {}",
1379                                 rd, prefix, nextHop);
1380                         fibManager.refreshVrfEntry(primaryRd, prefix);
1381                         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1382                         Optional<Prefixes> prefixToInterface = VpnUtil.getPrefixToInterface(dataBroker, vpnId, nextHop);
1383                         if (prefixToInterface.isPresent()) {
1384                             writeConfigTxn.delete(LogicalDatastoreType.OPERATIONAL,
1385                                     VpnUtil.getAdjacencyIdentifier(prefixToInterface.get().getVpnInterfaceName(),
1386                                             prefix));
1387                         }
1388                         LOG.info("VPN WITHDRAW: removePrefixFromBGP: Removed Fib Entry rd {} prefix {} nexthop {}",
1389                                 rd, prefix, tunnelIp);
1390                         return;
1391                     }
1392                 }
1393                 fibManager.removeOrUpdateFibEntry(dataBroker, primaryRd, prefix, tunnelIp, writeConfigTxn);
1394                 if (VpnUtil.isEligibleForBgp(rd, vpnName, dpnId, null /*networkName*/)) {
1395                     // TODO: Might be needed to include nextHop here
1396                     bgpManager.withdrawPrefix(rd, prefix);
1397                 }
1398             }
1399             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", rd, prefix,
1400                     nextHop);
1401         } catch (Exception e) {
1402             LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix);
1403         }
1404     }
1405
1406     @Override
1407     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1408         final VpnInterface update) {
1409         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1410                 update.getDpnId(), original.getVpnInstanceName(), update.getVpnInstanceName());
1411         final String vpnInterfaceName = update.getName();
1412         final String oldVpnName = original.getVpnInstanceName();
1413         final String newVpnName = update.getVpnInstanceName();
1414         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1415         final UpdateData updateData = new UpdateData(identifier, original, update);
1416         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1417         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency()
1418             != null) ? origAdjs.getAdjacency() : new ArrayList<>();
1419         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1420         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency()
1421             != null) ? updateAdjs.getAdjacency() : new ArrayList<>();
1422         //handles switching between <internal VPN - external VPN>
1423         if (oldVpnName != null && !oldVpnName.equals(newVpnName)) {
1424             vpnInterfacesUpdateQueue.add(updateData);
1425             LOG.info("update: UpdateData on VPNInterface {} on dpn {} update upon VPN swap from oldVpn {} to newVpn {}"
1426                     + " added to update queue", updateData.getOriginal().getName(), dpnId, oldVpnName,
1427                     newVpnName);
1428             return;
1429         }
1430         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1431         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1432             final DataStoreJobCoordinator vpnInfAdjUpdateDataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1433             vpnInfAdjUpdateDataStoreCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1434                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1435                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1436                 LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1437                         update.getName(), update.getVpnInstanceName());
1438                 //handle both addition and removal of adjacencies
1439                 //currently, new adjacency may be an extra route
1440                 if (!oldAdjs.equals(newAdjs)) {
1441                     for (Adjacency adj : newAdjs) {
1442                         if (oldAdjs.contains(adj)) {
1443                             oldAdjs.remove(adj);
1444                         } else {
1445                             // add new adjacency - right now only extra route will hit this path
1446                             addNewAdjToVpnInterface(identifier, primaryRd, adj, dpnId, writeOperTxn,
1447                                     writeConfigTxn);
1448                             LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to vpn "
1449                                             + "interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1450                                     adj.getLabel(), adj.getSubnetId(), update.getName(), update.getVpnInstanceName());
1451                         }
1452                     }
1453                     for (Adjacency adj : oldAdjs) {
1454                         delAdjFromVpnInterface(identifier, adj, dpnId, writeOperTxn, writeConfigTxn);
1455                         LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from vpn interface {}"
1456                                         + " on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1457                                 adj.getSubnetId(), update.getName(), update.getVpnInstanceName());
1458                     }
1459                 }
1460                 ListenableFuture<Void> operFuture = writeOperTxn.submit();
1461                 try {
1462                     operFuture.get();
1463                 } catch (ExecutionException e) {
1464                     LOG.error("Exception encountered while submitting operational future for update"
1465                             + " VpnInterface {}: {}", vpnInterfaceName, e);
1466                     return null;
1467                 }
1468                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1469                 futures.add(writeConfigTxn.submit());
1470                 LOG.info("update: vpn interface updated for interface {} oldVpn {} newVpn {} processed successfully",
1471                         update.getName(), original.getVpnInstanceName(), update.getVpnInstanceName());
1472                 return futures;
1473             });
1474         } else {
1475             LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1476                     + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1477         }
1478     }
1479
1480     class VpnInterfaceUpdateTimerTask extends TimerTask {
1481         private final Logger log = LoggerFactory.getLogger(VpnInterfaceUpdateTimerTask.class);
1482
1483         @Override
1484         public void run() {
1485             List<UpdateData> processQueue = new ArrayList<>();
1486             List<VpnInterface> vpnInterfaceList = new ArrayList<>();
1487             vpnInterfacesUpdateQueue.drainTo(processQueue);
1488
1489             for (UpdateData updData : processQueue) {
1490                 log.info("run: VPN Interface update event - intfName {} onto vpnName {} running config-driven swap"
1491                         + " removal", updData.getOriginal().getName(), updData.getOriginal().getVpnInstanceName());
1492                 remove(updData.getIdentifier(), updData.getOriginal());
1493                 log.info("run: Processed Remove for update on VPNInterface {} upon VPN swap from oldVpn {}"
1494                         + " to newVpn {}", updData.getOriginal().getName(), updData.getOriginal().getVpnInstanceName(),
1495                         updData.getUpdate().getVpnInstanceName());
1496                 vpnInterfaceList.add(updData.getOriginal());
1497             }
1498
1499             /* Decide the max-wait time based on number of VpnInterfaces.
1500             *  max-wait-time is num-of-interface * 4seconds (random choice).
1501             *  Every 2sec poll VpnToDpnList. If VpnInterface is removed ,
1502             *  remove it from vpnInterfaceList.
1503             */
1504
1505             int maxWaitTime =
1506                 vpnInterfaceList.size() * (int) (VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS / 1000);
1507             int waitTime = 2;
1508             Iterator<VpnInterface> vpnInterfaceIterator = vpnInterfaceList.iterator();
1509             VpnInterface vpnInterface;
1510             while (waitTime < maxWaitTime) {
1511                 try {
1512                     Thread.sleep(2000); // sleep for 2sec
1513                 } catch (InterruptedException e) {
1514                     // Ignored
1515                 }
1516
1517                 while (vpnInterfaceIterator.hasNext()) {
1518                     vpnInterface = vpnInterfaceIterator.next();
1519                     if (!VpnUtil.isVpnIntfPresentInVpnToDpnList(dataBroker, vpnInterface)) {
1520                         vpnInterfaceIterator.remove();
1521                     }
1522                 }
1523                 if (vpnInterfaceList.size() == 0) {
1524                     log.info("run: All VpnInterfaces are successfully removed from OLD VPN after time {}", waitTime);
1525                     break;
1526                 }
1527                 waitTime += 2; //Increment linearly by 2sec.
1528             }
1529
1530             if (vpnInterfaceList.size() > 0) {
1531                 log.error("run: VpnInterfacesList not removed from old Vpn even after waiting {}", waitTime);
1532             }
1533
1534             for (UpdateData updData : processQueue) {
1535                 if (vpnInterfaceList.contains(updData.getOriginal())) {
1536                     log.error("run: Failed to swap VpnInterface {} from oldVpn {} to target VPN {} as it has not been"
1537                             + " cleaned up from the oldVpn", updData.getOriginal().getName(),
1538                             updData.getOriginal().getVpnInstanceName(),  updData.getUpdate().getVpnInstanceName());
1539                     continue;
1540                 }
1541                 log.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven"
1542                         + " swap addition", updData.getUpdate().getName(),
1543                         updData.getUpdate().getVpnInstanceName());
1544                 final Adjacencies origAdjs = updData.getOriginal().getAugmentation(Adjacencies.class);
1545                 final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1546                     ? origAdjs.getAdjacency() : new ArrayList<>();
1547                 final Adjacencies updateAdjs = updData.getUpdate().getAugmentation(Adjacencies.class);
1548                 final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1549                     ? updateAdjs.getAdjacency() : new ArrayList<>();
1550                 addVpnInterface(updData.getIdentifier(), updData.getUpdate(), oldAdjs, newAdjs);
1551                 log.info("run: Processed Add for update on VPNInterface {} from oldVpn {} to newVpn {} upon VPN swap",
1552                         updData.getUpdate().getName(), updData.getOriginal().getVpnInstanceName(),
1553                         updData.getUpdate().getVpnInstanceName());
1554             }
1555         }
1556     }
1557
1558     private String getErrorText(Collection<RpcError> errors) {
1559         StringBuilder errorText = new StringBuilder();
1560         for (RpcError error : errors) {
1561             errorText.append(",").append(error.getErrorType()).append("-")
1562                 .append(error.getMessage());
1563         }
1564         return errorText.toString();
1565     }
1566
1567     //TODO (KIRAN) : Move to implemetation specific L3vpnOverMplsGrePopulator
1568     public void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1569                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd) {
1570         Preconditions.checkNotNull(label, "addToLabelMapper: label cannot be null or empty!");
1571         Preconditions.checkNotNull(prefix, "addToLabelMapper: prefix cannot be null or empty!");
1572         Preconditions.checkNotNull(vpnId, "addToLabelMapper: vpnId cannot be null or empty!");
1573         Preconditions.checkNotNull(rd, "addToLabelMapper: rd cannot be null or empty!");
1574         if (!isSubnetRoute) {
1575             // NextHop must be present for non-subnetroute entries
1576             Preconditions.checkNotNull(nextHopIpList, "addToLabelMapper: nextHopIp cannot be null or empty!");
1577         }
1578         synchronized (label.toString().intern()) {
1579             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1580             LOG.info("addToLabelMapper: label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}"
1581                     + " elanTag {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd, elanTag);
1582             if (dpnId != null) {
1583                 LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1584                 lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList)
1585                     .setParentVpnid(vpnId).setIsSubnetRoute(isSubnetRoute);
1586                 if (elanTag != null) {
1587                     lriBuilder.setElanTag(elanTag);
1588                 } else {
1589                     LOG.warn("addToLabelMapper: elanTag is null for label {} prefix {} rd {} vpnId {}",
1590                             label, prefix, rd, vpnId);
1591                 }
1592                 if (vpnInterfaceName != null) {
1593                     lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1594                 } else {
1595                     LOG.warn("addToLabelMapper: vpn interface is null for label {} prefix {} rd {} vpnId {}",
1596                             label, prefix, rd, vpnId);
1597                 }
1598                 lriBuilder.setParentVpnRd(rd);
1599                 VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
1600                 if (vpnInstanceOpDataEntry != null) {
1601                     List<String> vpnInstanceNames = Collections
1602                             .singletonList(vpnInstanceOpDataEntry.getVpnInstanceName());
1603                     lriBuilder.setVpnInstanceList(vpnInstanceNames);
1604                 }
1605                 LabelRouteInfo lri = lriBuilder.build();
1606                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1607                         .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) label)).build();
1608                 tx.merge(LogicalDatastoreType.OPERATIONAL, lriIid, lri, true);
1609                 tx.submit();
1610                 LOG.info("addToLabelMapper: Added label route info to label {} prefix {} nextHopList {} vpnId {}"
1611                         + " interface {} rd {} elantag {}", label, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd,
1612                         elanTag);
1613             } else {
1614                 LOG.error("addToLabelMapper: Can't add entry to label map for label {} prefix {} nextHopList {}"
1615                         + " vpnId {} interface {} rd {} elantag {} ,dpnId is null", label, prefix, nextHopIpList,
1616                         vpnId, vpnInterfaceName, rd, elanTag);
1617             }
1618         }
1619     }
1620
1621     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1622         Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1623         synchronized (label.toString().intern()) {
1624             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1625                     .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) label)).build();
1626             Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1627             if (opResult.isPresent()) {
1628                 LabelRouteInfo labelRouteInfo =
1629                     new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1630                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1631             }
1632         }
1633         LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList);
1634     }
1635
1636     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1637         SubnetRoute route, WriteTransaction writeConfigTxn) {
1638
1639         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1640         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, null /* parentVpnRd */)
1641                 .addAugmentation(SubnetRoute.class, route).build();
1642         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1643         InstanceIdentifierBuilder<VrfTables> idBuilder =
1644             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1645         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1646         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1647         if (writeConfigTxn != null) {
1648             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1649         } else {
1650             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1651         }
1652         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1653                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1654     }
1655
1656     public void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName) {
1657         fibManager.removeFibEntry(dataBroker, rd, prefix, null);
1658         List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1659         for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
1660             String importingRd = vpnInstance.getVrfId();
1661             fibManager.removeFibEntry(dataBroker, importingRd, prefix, null);
1662             LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Deleted imported subnet route rd {} prefix {}"
1663                     + " from vpn {} importingRd {}", rd, prefix, vpnInstance.getVpnInstanceName(), importingRd);
1664         }
1665         LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Removed subnetroute FIB for prefix {} rd {}"
1666                 + " vpnName {}", prefix, rd, vpnName);
1667     }
1668
1669     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, String primaryRd,
1670                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1671                                            WriteTransaction writeConfigTxn) {
1672
1673         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1674
1675         if (optVpnInterface.isPresent()) {
1676             VpnInterface currVpnIntf = optVpnInterface.get();
1677             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1678             String vpnName = currVpnIntf.getVpnInstanceName();
1679             VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1680             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1681             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, adjPath);
1682             boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1683             VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1684             long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1685             VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1686             List<Adjacency> adjacencies;
1687             if (optAdjacencies.isPresent()) {
1688                 adjacencies = optAdjacencies.get().getAdjacency();
1689             } else {
1690                 // This code will be hit in case of first PNF adjacency
1691                 adjacencies = new ArrayList<>();
1692             }
1693             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1694             L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1695                     .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1696             Adjacency operationalAdjacency = null;
1697             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) {
1698                 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1699                         : RouteOrigin.STATIC;
1700                 String nh = adj.getNextHopIpList().get(0);
1701                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1702                 synchronized (vpnPrefixKey.intern()) {
1703                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1704                             dataBroker, vpnId, null, prefix, vpnName, nh, dpnId, writeOperTxn);
1705                     if (rdToAllocate.isPresent()) {
1706                         input.setRd(rdToAllocate.get());
1707                         operationalAdjacency = populator.createOperationalAdjacency(input);
1708                         int label = operationalAdjacency.getLabel().intValue();
1709                         addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1710                                 currVpnIntf.getVpnInstanceName(), label, l3vni, origin,
1711                                 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1712                         LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1713                                 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1714                                 currVpnIntf.getVpnInstanceName(), label, currVpnIntf.getName(), dpnId);
1715                     } else {
1716                         LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName,
1717                                 prefix);
1718                         return;
1719                     }
1720                     // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1721                     // Keeping the MPLS check for now.
1722                     if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1723                         final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1724                         List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1725                         vpnsToImportRoute.forEach(vpn -> {
1726                             if (vpn.getVrfId() != null) {
1727                                 VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1728                                         dataBroker, vpn.getVpnId(), vpnId, prefix,
1729                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId,
1730                                         writeOperTxn)
1731                                         .ifPresent(
1732                                             rds -> addExtraRoute(VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1733                                                     adj.getIpAddress(), nh, rds,
1734                                                     currVpnIntf.getVpnInstanceName(),
1735                                                     opAdjacency.getLabel().intValue(),
1736                                                     l3vni, RouteOrigin.SELF_IMPORTED,
1737                                                     currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn));
1738                             }
1739                         });
1740                     }
1741                 }
1742             } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1743                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1744                         currVpnIntf.getName(), vpnName);
1745
1746                 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1747
1748                 writeOperTxn.merge(
1749                         LogicalDatastoreType.OPERATIONAL,
1750                         VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1751                                 adj.getSubnetId().getValue()), prefix),
1752                         VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1753                                 adj.getSubnetId(), true /* isNatPrefix */), true);
1754
1755                 fibManager.addOrUpdateFibEntry(dataBroker, adj.getSubnetId().getValue(), adj.getMacAddress(),
1756                         adj.getIpAddress(), Collections.EMPTY_LIST, null /* EncapType */, 0 /* label */, 0 /*l3vni*/,
1757                       null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1758
1759                 input.setRd(adj.getVrfId());
1760             }
1761             if (operationalAdjacency == null) {
1762                 operationalAdjacency = populator.createOperationalAdjacency(input);
1763             }
1764             adjacencies.add(operationalAdjacency);
1765             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1766             VpnInterface newVpnIntf =
1767                 VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId,
1768                     currVpnIntf.isScheduledForRemove());
1769
1770             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1771         }
1772     }
1773
1774     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1775         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1776         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1777     }
1778
1779     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId,
1780             WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1781         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1782
1783         if (optVpnInterface.isPresent()) {
1784             VpnInterface currVpnIntf = optVpnInterface.get();
1785
1786             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1787             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1788             if (optAdjacencies.isPresent()) {
1789                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1790
1791                 if (!adjacencies.isEmpty()) {
1792                     LOG.trace("delAdjFromVpnInterface: Adjacencies are " + adjacencies);
1793                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1794                     while (adjIt.hasNext()) {
1795                         Adjacency adjElem = adjIt.next();
1796                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1797                             String rd = adjElem.getVrfId();
1798                             adjIt.remove();
1799
1800                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1801                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1802                                     currVpnIntf.getVpnInstanceName(),
1803                                     aug, dpnId, currVpnIntf.isScheduledForRemove());
1804
1805                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1806                             if (adj.getNextHopIpList() != null) {
1807                                 for (String nh : adj.getNextHopIpList()) {
1808                                     deleteExtraRouteFromCurrentAndImportingVpns(currVpnIntf.getVpnInstanceName(),
1809                                             adj.getIpAddress(), nh, rd, currVpnIntf.getName(), writeConfigTxn);
1810                                 }
1811                             } else if (adj.isPhysNetworkFunc()) {
1812                                 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet [}",
1813                                         adj.getIpAddress(), adj.getSubnetId());
1814                                 fibManager.removeFibEntry(dataBroker, adj.getSubnetId().getValue(), adj.getIpAddress(),
1815                                         writeConfigTxn);
1816                             }
1817                             break;
1818                         }
1819
1820                     }
1821                 }
1822                 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1823                         dpnId, adj.getVrfId());
1824             } else {
1825                 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1826                         + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1827             }
1828         }
1829     }
1830
1831     protected void addExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
1832                                  int label, Long l3vni, RouteOrigin origin, String intfName, Adjacency operationalAdj,
1833                                  VrfEntry.EncapType encapType, WriteTransaction writeConfigTxn) {
1834
1835         Boolean writeConfigTxnPresent = true;
1836         if (writeConfigTxn == null) {
1837             writeConfigTxnPresent = false;
1838             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1839         }
1840
1841         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1842         VpnUtil.syncUpdate(
1843                 dataBroker,
1844                 LogicalDatastoreType.OPERATIONAL,
1845                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, (rd != null) ? rd : routerID,
1846                         destination),
1847                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
1848
1849         BigInteger dpnId = null;
1850         if (intfName != null && !intfName.isEmpty()) {
1851             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1852             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1853             if (nextHopIp == null || nextHopIp.isEmpty()) {
1854                 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
1855                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
1856                         dpnId);
1857                 return;
1858             }
1859             nextHop = nextHopIp;
1860         }
1861
1862         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
1863         List<String> nextHopIpList = Collections.singletonList(nextHop);
1864
1865         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
1866         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1867         // first place.
1868         Optional<InterVpnLinkDataComposite> optVpnLink = InterVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
1869         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
1870             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
1871             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1872             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1873             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1874             // This is like leaking one of the Vpn2 routes towards Vpn1
1875             String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
1876             String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
1877             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
1878             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1879                                                 VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1880             if (newLabel == 0) {
1881                 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
1882                         + " route for destination {}", destination);
1883                 return;
1884             }
1885             ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC,
1886                                       NwConstants.ADD_FLOW);
1887         } else {
1888             Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
1889                     .getVpnExtraroutes(dataBroker, vpnName, (rd != null) ? rd : routerID, destination);
1890             if (optVpnExtraRoutes.isPresent()) {
1891                 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
1892                 if (nhList != null && nhList.size() > 1) {
1893                     // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
1894                     fibManager.refreshVrfEntry(primaryRd, destination);
1895                 } else {
1896                     L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop).setL3vni(l3vni)
1897                             .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
1898                             .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
1899                     L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, writeConfigTxn, null);
1900                 }
1901             }
1902         }
1903
1904         if (!writeConfigTxnPresent) {
1905             writeConfigTxn.submit();
1906         }
1907     }
1908
1909     protected void delExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
1910         String intfName, WriteTransaction writeConfigTxn) {
1911         Boolean writeConfigTxnPresent = true;
1912         BigInteger dpnId = null;
1913         if (writeConfigTxn == null) {
1914             writeConfigTxnPresent = false;
1915             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1916         }
1917         String tunnelIp = nextHop;
1918         if (intfName != null && !intfName.isEmpty()) {
1919             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1920             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1921             if (nextHopIp == null || nextHopIp.isEmpty()) {
1922                 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
1923                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
1924                         dpnId);
1925             }
1926             tunnelIp = nextHopIp;
1927         }
1928         if (rd != null) {
1929             String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1930             removePrefixFromBGP(primaryRd, rd, vpnName, destination, nextHop, tunnelIp, dpnId, writeConfigTxn);
1931             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
1932         } else {
1933             // add FIB route directly
1934             fibManager.removeOrUpdateFibEntry(dataBroker, routerID, destination, tunnelIp, writeConfigTxn);
1935             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
1936                     routerID);
1937         }
1938         if (!writeConfigTxnPresent) {
1939             writeConfigTxn.submit();
1940         }
1941     }
1942
1943     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1944         String rd, String intfName, WriteTransaction writeConfigTxn) {
1945         delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1946         List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1947         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1948             String vpnRd = vpn.getVrfId();
1949             if (vpnRd != null) {
1950                 delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1951             }
1952         }
1953     }
1954
1955     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1956         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1957             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1958             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1959     }
1960
1961     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1962         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1963             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1964     }
1965
1966     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1967         WriteTransaction writeOperTxn) {
1968         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1969         if (dpId.equals(BigInteger.ZERO)) {
1970             LOG.error("addToNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1971                     + " association model", vpnInterfaceName, routerName);
1972
1973             return;
1974         }
1975         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1976
1977         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1978             .OPERATIONAL, routerDpnListIdentifier);
1979         RouterInterfaces routerInterface =
1980             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1981                 vpnInterfaceName).build();
1982         if (optionalRouterDpnList.isPresent()) {
1983             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1984                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1985         } else {
1986             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1987             builder.setRouterId(routerName);
1988             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1989             List<RouterInterfaces> routerInterfaces = new ArrayList<>();
1990             routerInterfaces.add(routerInterface);
1991             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
1992             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1993                 getRouterId(routerName),
1994                 builder.build(), true);
1995         }
1996     }
1997
1998     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1999         WriteTransaction writeOperTxn) {
2000         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
2001         if (dpId.equals(BigInteger.ZERO)) {
2002             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
2003                     + " dissociation model", vpnInterfaceName, routerName);
2004
2005             return;
2006         }
2007         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2008         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
2009             .OPERATIONAL, routerDpnListIdentifier);
2010         if (optionalRouterDpnList.isPresent()) {
2011             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
2012             RouterInterfaces routerInterface =
2013                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
2014                     vpnInterfaceName).build();
2015
2016             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
2017                 if (routerInterfaces.isEmpty()) {
2018                     if (writeOperTxn != null) {
2019                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2020                     } else {
2021                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2022                     }
2023                 } else {
2024                     if (writeOperTxn != null) {
2025                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2026                             RouterInterfaces.class,
2027                             new RouterInterfacesKey(vpnInterfaceName)));
2028                     } else {
2029                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
2030                             routerDpnListIdentifier.child(
2031                                 RouterInterfaces.class,
2032                                 new RouterInterfacesKey(vpnInterfaceName)));
2033                     }
2034                 }
2035             }
2036         }
2037     }
2038
2039     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
2040         WriteTransaction writeOperTxn) {
2041         if (dpId.equals(BigInteger.ZERO)) {
2042             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
2043                     + " dissociation model", vpnInterfaceName, routerName);
2044
2045             return;
2046         }
2047         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2048         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
2049             .OPERATIONAL, routerDpnListIdentifier);
2050         if (optionalRouterDpnList.isPresent()) {
2051             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
2052             RouterInterfaces routerInterface =
2053                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
2054                     vpnInterfaceName).build();
2055             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
2056                 if (routerInterfaces.isEmpty()) {
2057                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2058                 } else {
2059                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2060                         RouterInterfaces.class,
2061                         new RouterInterfacesKey(vpnInterfaceName)));
2062                 }
2063             }
2064         }
2065     }
2066
2067     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2068                                                     WriteTransaction writeConfigTxn) {
2069         if (vpnInterface == null) {
2070             return;
2071         }
2072         String vpnName = vpnInterface.getVpnInstanceName();
2073         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
2074         if (adjs == null) {
2075             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2076                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2077             return;
2078         }
2079         for (Adjacency adj : adjs) {
2080             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2081                 String primaryInterfaceIp = adj.getIpAddress();
2082                 String macAddress = adj.getMacAddress();
2083                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2084
2085                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
2086                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2087
2088                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2089                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2090                 fibManager.addFibEntryForRouterInterface(dataBroker, primaryRd, prefix,
2091                         routerInt, label, writeConfigTxn);
2092                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2093                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2094                         macAddress);
2095                 return;
2096             }
2097         }
2098         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2099                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
2100                 primaryRd, vpnName);
2101     }
2102
2103     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface, WriteTransaction writeConfigTxn) {
2104         List<Adjacency> adjsList = new ArrayList<>();
2105         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
2106         String rd = VpnUtil.getVpnRd(dataBroker, vpnInterface.getVpnInstanceName());
2107         if (adjs != null) {
2108             adjsList = adjs.getAdjacency();
2109             for (Adjacency adj : adjsList) {
2110                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2111                     String primaryInterfaceIp = adj.getIpAddress();
2112                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2113                     fibManager.removeFibEntry(dataBroker, rd, prefix, writeConfigTxn);
2114                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2115                             + " prefix {}", vpnInterface.getName(), vpnInterface.getVpnInstanceName(), rd, prefix);
2116                     return;
2117                 }
2118             }
2119         } else {
2120             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2121                     vpnInterface.getName(), rd);
2122         }
2123     }
2124
2125     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData) {
2126         addVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, null, null);
2127     }
2128
2129     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2130         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2131                 .get(vpnInterface.getVpnInstanceName());
2132         if (vpnInterfaces == null) {
2133             vpnInterfaces = new ConcurrentLinkedQueue<>();
2134         }
2135         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2136         unprocessedVpnInterfaces.put(vpnInterface.getVpnInstanceName(), vpnInterfaces);
2137         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2138                 vpnInterface.getName(), vpnInterface.getVpnInstanceName());
2139     }
2140
2141     public boolean isVpnInstanceReady(String vpnInstanceName) {
2142         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2143         if (vpnRd == null) {
2144             return false;
2145         }
2146         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2147
2148         return (vpnInstanceOpDataEntry != null);
2149     }
2150
2151     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2152         synchronized (vpnInstanceName.intern()) {
2153             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2154                     unprocessedVpnInterfaces.get(vpnInstanceName);
2155             if (vpnInterfaces != null) {
2156                 while (!vpnInterfaces.isEmpty()) {
2157                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2158                     if (hasVpnInstanceCreatedSuccessfully) {
2159                         processSavedInterface(savedInterface);
2160                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2161                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2162                     } else {
2163                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2164                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2165                     }
2166                 }
2167             } else {
2168                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2169             }
2170         }
2171     }
2172
2173     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2174             VpnInterface vpnInterface) {
2175         synchronized (vpnInterface.getVpnInstanceName().intern()) {
2176             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2177                     unprocessedVpnInterfaces.get(vpnInterface.getVpnInstanceName());
2178             if (vpnInterfaces != null) {
2179                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2180                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2181                             + "unprocessed list", vpnInterface.getName(), vpnInterface.getVpnInstanceName());
2182                 }
2183             } else {
2184                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2185                         vpnInterface.getVpnInstanceName());
2186             }
2187         }
2188     }
2189
2190     public void vpnInstanceIsReady(String vpnInstanceName) {
2191         processSavedInterfaces(vpnInstanceName, true);
2192     }
2193
2194     public void vpnInstanceFailed(String vpnInstanceName) {
2195         processSavedInterfaces(vpnInstanceName, false);
2196     }
2197
2198     private static class UnprocessedVpnInterfaceData {
2199         InstanceIdentifier<VpnInterface> identifier;
2200         VpnInterface vpnInterface;
2201
2202         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2203             super();
2204             this.identifier = identifier;
2205             this.vpnInterface = vpnInterface;
2206         }
2207
2208         @Override
2209         public int hashCode() {
2210             final int prime = 31;
2211             int result = 1;
2212             result = prime * result + ((identifier == null) ? 0 : identifier.hashCode());
2213             result = prime * result + ((vpnInterface == null) ? 0 : vpnInterface.hashCode());
2214             return result;
2215         }
2216
2217         @Override
2218         public boolean equals(Object obj) {
2219             if (this == obj) {
2220                 return true;
2221             }
2222             if (obj == null) {
2223                 return false;
2224             }
2225             if (getClass() != obj.getClass()) {
2226                 return false;
2227             }
2228             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2229             if (identifier == null) {
2230                 if (other.identifier != null) {
2231                     return false;
2232                 }
2233             } else if (!identifier.equals(other.identifier)) {
2234                 return false;
2235             }
2236             if (vpnInterface == null) {
2237                 if (other.vpnInterface != null) {
2238                     return false;
2239                 }
2240             } else if (!vpnInterface.equals(other.vpnInterface)) {
2241                 return false;
2242             }
2243             return true;
2244         }
2245     }
2246
2247     public void updateVpnInterfacesForUnProcessAdjancencies(DataBroker dataBroker,
2248                                                                    String vpnName) {
2249         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2250         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2251         if (vpnInstanceOpData == null) {
2252             return;
2253         }
2254         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2255         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2256             return;
2257         }
2258         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2259         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2260             InstanceIdentifier<VpnInterface> existingVpnInterfaceId =
2261                     VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getInterfaceName());
2262             Optional<VpnInterface> vpnInterfaceOptional = VpnUtil.read(dataBroker,
2263                     LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2264             if (!vpnInterfaceOptional.isPresent()) {
2265                 return;
2266             }
2267             List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2268                     vpnInterface.getInterfaceName());
2269             if (configVpnAdjacencies == null) {
2270                 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2271                 return;
2272             }
2273             List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2274                     .getAugmentation(Adjacencies.class).getAdjacency();
2275             // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2276             // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2277             // in operational DS. These unprocessed adjacencies will be handled below.
2278             // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2279             // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2280             configVpnAdjacencies.stream()
2281                     .filter(adjacency -> operationVpnAdjacencies.stream()
2282                             .noneMatch(operationalAdjacency ->
2283                                     operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2284                     .forEach(adjacency -> {
2285                         LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2286                         DataStoreJobCoordinator dataStoreJobCoordinator = DataStoreJobCoordinator.getInstance();
2287                         dataStoreJobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2288                             () -> {
2289                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
2290                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
2291                                 addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd, adjacency,
2292                                             vpnInterfaceOptional.get().getDpnId(), writeConfigTxn, writeOperTxn);
2293                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
2294                                 ListenableFuture<Void> operFuture = writeOperTxn.submit();
2295                                 try {
2296                                     operFuture.get();
2297                                 } catch (ExecutionException | InterruptedException e) {
2298                                     LOG.error("Exception encountered while submitting operational"
2299                                             + " future for vpnInterface {}", vpnInterface, e);
2300                                 }
2301                                 futures.add(writeConfigTxn.submit());
2302                                 return futures;
2303                             });
2304                     });
2305         }));
2306     }
2307 }