vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.ListenableFuture;
  14
  15 import java.math.BigInteger;
  16 import java.util.ArrayList;
  17 import java.util.Collection;
  18 import java.util.Collections;
  19 import java.util.HashSet;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.Set;
  25 import java.util.TimerTask;
  26 import java.util.concurrent.BlockingQueue;
  27 import java.util.concurrent.ConcurrentHashMap;
  28 import java.util.concurrent.ConcurrentLinkedQueue;
  29 import java.util.concurrent.ExecutionException;
  30 import java.util.concurrent.Executors;
  31 import java.util.concurrent.LinkedBlockingQueue;
  32 import java.util.concurrent.ScheduledThreadPoolExecutor;
  33 import java.util.concurrent.TimeUnit;
  34 import java.util.function.Predicate;
  35 import java.util.stream.Collectors;
  36
  37 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  38 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  41 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  42 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  43 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  47 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  48 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  49 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  50 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  51 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  52 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  53 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  54 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderUtil;
  55 import org.opendaylight.netvirt.vpnmanager.intervpnlink.InterVpnLinkUtil;
  56 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  57 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  58 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  59 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnTargets;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 108 import org.opendaylight.yangtools.yang.common.RpcError;
 109 import org.slf4j.Logger;
 110 import org.slf4j.LoggerFactory;
 111
 112 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager>
 113     implements AutoCloseable {
 114
 115     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 116     private static final int VPN_INF_UPDATE_TIMER_TASK_DELAY = 1000;
 117     private static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
 118
 119     private final DataBroker dataBroker;
 120     private final IBgpManager bgpManager;
 121     private final IFibManager fibManager;
 122     private final IMdsalApiManager mdsalManager;
 123     private final IdManagerService idManager;
 124     private final OdlArputilService arpManager;
 125     private final OdlInterfaceRpcService ifaceMgrRpcService;
 126     private final VpnFootprintService vpnFootprintService;
 127     private final IInterfaceManager interfaceManager;
 128     private final IVpnManager vpnManager;
 129
 130     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 131
 132     private BlockingQueue<UpdateData> vpnInterfacesUpdateQueue = new LinkedBlockingQueue<>();
 133     private ScheduledThreadPoolExecutor vpnInfUpdateTaskExecutor = (ScheduledThreadPoolExecutor) Executors
 134             .newScheduledThreadPool(1);
 135
 136     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 137             new ConcurrentHashMap<>();
 138
 139     public VpnInterfaceManager(final DataBroker dataBroker,
 140                                final IBgpManager bgpManager,
 141                                final OdlArputilService arpManager,
 142                                final IdManagerService idManager,
 143                                final IMdsalApiManager mdsalManager,
 144                                final IFibManager fibManager,
 145                                final OdlInterfaceRpcService ifaceMgrRpcService,
 146                                final VpnFootprintService vpnFootprintService,
 147                                final IInterfaceManager interfaceManager,
 148                                final IVpnManager vpnManager) {
 149         super(VpnInterface.class, VpnInterfaceManager.class);
 150
 151         this.dataBroker = dataBroker;
 152         this.bgpManager = bgpManager;
 153         this.arpManager = arpManager;
 154         this.idManager = idManager;
 155         this.mdsalManager = mdsalManager;
 156         this.fibManager = fibManager;
 157         this.ifaceMgrRpcService = ifaceMgrRpcService;
 158         this.vpnFootprintService = vpnFootprintService;
 159         this.interfaceManager = interfaceManager;
 160         this.vpnManager = vpnManager;
 161         vpnInfUpdateTaskExecutor.scheduleWithFixedDelay(new VpnInterfaceUpdateTimerTask(),
 162             0, VPN_INF_UPDATE_TIMER_TASK_DELAY, TIME_UNIT);
 163     }
 164
 165     public Runnable isNotifyTaskQueued(String intfName) {
 166         return vpnIntfMap.remove(intfName);
 167     }
 168
 169     public void start() {
 170         LOG.info("{} start", getClass().getSimpleName());
 171         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 172     }
 173
 174     @Override
 175     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 176         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 177     }
 178
 179     @Override
 180     protected VpnInterfaceManager getDataTreeChangeListener() {
 181         return VpnInterfaceManager.this;
 182     }
 183
 184     private InstanceIdentifier<Interface> getInterfaceListenerPath() {
 185         return InstanceIdentifier.create(InterfacesState.class).child(Interface.class);
 186     }
 187
 188     @Override
 189     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 190         if (canHandleNewVpnInterface(identifier, vpnInterface)) {
 191             addVpnInterface(identifier, vpnInterface, null, null);
 192         }
 193     }
 194
 195     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 196             final VpnInterface vpnInterface) {
 197         synchronized (vpnInterface.getVpnInstanceName().intern()) {
 198             if (isVpnInstanceReady(vpnInterface.getVpnInstanceName())) {
 199                 return true;
 200             }
 201             addToUnprocessedVpnInterfaces(identifier, vpnInterface);
 202             return false;
 203         }
 204     }
 205
 206     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 207         final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 208         LOG.trace("VPN Interface add event - key: {}, value: {}", identifier, vpnInterface);
 209         LOG.info("VPN Interface add event - intfName {}", vpnInterface.getName());
 210         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 211         final String interfaceName = key.getName();
 212
 213         Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 214         if (interfaceState != null) {
 215             final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 216             final int ifIndex = interfaceState.getIfIndex();
 217             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 218             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
 219                 () -> {
 220                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 221                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 222                     WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 223                     processVpnInterfaceUp(dpnId, vpnInterface, ifIndex, false, writeConfigTxn, writeOperTxn,
 224                             writeInvTxn, interfaceState);
 225                     if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 226                         LOG.trace("Adjacency changed upon VPNInterface {} Update for swapping VPN case",
 227                                 interfaceName);
 228                         if (newAdjs != null) {
 229                             for (Adjacency adj : newAdjs) {
 230                                 if (oldAdjs.contains(adj)) {
 231                                     oldAdjs.remove(adj);
 232                                 } else {
 233                                     addNewAdjToVpnInterface(identifier, adj, dpnId, writeOperTxn,
 234                                             writeConfigTxn);
 235                                 }
 236                             }
 237                         }
 238                         for (Adjacency adj : oldAdjs) {
 239                             delAdjFromVpnInterface(identifier, adj, dpnId, writeOperTxn, writeConfigTxn);
 240                         }
 241                     }
 242                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
 243                     try {
 244                         operFuture.get();
 245                     } catch (ExecutionException e) {
 246                         LOG.error("Exception encountered while submitting operational future for addVpnInterface {}: "
 247                                 + "{}", vpnInterface.getName(), e);
 248                         return null;
 249                     }
 250                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 251                     futures.add(writeConfigTxn.submit());
 252                     futures.add(writeInvTxn.submit());
 253                     return futures;
 254                 });
 255         } else if (Boolean.TRUE.equals(vpnInterface.isIsRouterInterface())) {
 256             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 257             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 258                 () -> {
 259                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 260                     createFibEntryForRouterInterface(vpnInterface, interfaceName, writeConfigTxn);
 261                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 262                     futures.add(writeConfigTxn.submit());
 263                     return futures;
 264                 });
 265         } else {
 266             LOG.error("Handling addition of VPN interface {} skipped as interfaceState is not available",
 267                     interfaceName);
 268         }
 269     }
 270
 271     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface,
 272             final int lportTag, boolean isInterfaceUp,
 273             WriteTransaction writeConfigTxn,
 274             WriteTransaction writeOperTxn,
 275             WriteTransaction writeInvTxn,
 276             Interface interfaceState) {
 277
 278         final String interfaceName = vpnInterface.getName();
 279         if (!isInterfaceUp) {
 280             final String vpnName = vpnInterface.getVpnInstanceName();
 281             LOG.info("Binding vpn service to interface {} ", interfaceName);
 282             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 283             if (vpnId == VpnConstants.INVALID_ID) {
 284                 LOG.error("VpnInstance to VPNId mapping not available for VpnName {} processing vpninterface {}"
 285                         + ", bailing out now.", vpnName, interfaceName);
 286                 return;
 287             }
 288
 289             boolean waitForVpnInterfaceOpRemoval = false;
 290             VpnInterface opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, vpnInterface.getName());
 291             if (opVpnInterface != null) {
 292                 String opVpnName = opVpnInterface.getVpnInstanceName();
 293                 String primaryInterfaceIp = null;
 294                 if (opVpnName.equals(vpnName)) {
 295                     // Please check if the primary VRF Entry does not exist for VPNInterface
 296                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 297                     // back to back
 298                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 299                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 300                     if (adjs == null) {
 301                         LOG.info(
 302                                 "VPN Interface {} addition failed as adjacencies for this vpn interface could not be "
 303                                         + "obtained", interfaceName);
 304                         return;
 305                     }
 306                     for (Adjacency adj : adjs) {
 307                         if (adj.isPrimaryAdjacency()) {
 308                             primaryInterfaceIp = adj.getIpAddress();
 309                             break;
 310                         }
 311                     }
 312                     if (primaryInterfaceIp == null) {
 313                         LOG.info("VPN Interface {} addition failed as primary adjacency "
 314                                 + "for this vpn interface could not be obtained", interfaceName);
 315                         return;
 316                     }
 317                     // Get the rd of the vpn instance
 318                     String primaryRd = VpnUtil.getPrimaryRd(dataBroker, opVpnName);
 319                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 320                     if (vrf != null) {
 321                         LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
 322                         return;
 323                     }
 324                     waitForVpnInterfaceOpRemoval = true;
 325                 } else {
 326                     LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
 327                             interfaceName, vpnName, opVpnName);
 328                 }
 329             }
 330             if (!waitForVpnInterfaceOpRemoval) {
 331                 // Add the VPNInterface and quit
 332                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 333                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/);
 334                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, interfaceName,
 335                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 336                 if (interfaceManager.isExternalInterface(interfaceName)) {
 337                     processExternalVpnInterface(vpnInterface, vpnId, dpId, lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 338                 }
 339                 return;
 340             }
 341
 342             // FIB didn't get a chance yet to clean up this VPNInterface
 343             // Let us give it a chance here !
 344             LOG.info("Trying to add VPN Interface {}, but waiting for FIB to clean up! ", interfaceName);
 345             try {
 346                 Runnable notifyTask = new VpnNotifyTask();
 347                 vpnIntfMap.put(interfaceName, notifyTask);
 348                 synchronized (notifyTask) {
 349                     try {
 350                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 351                     } catch (InterruptedException e) {
 352                         // Ignored
 353                     }
 354                 }
 355             } finally {
 356                 vpnIntfMap.remove(interfaceName);
 357             }
 358
 359             opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
 360             if (opVpnInterface != null) {
 361                 LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...",
 362                         interfaceName);
 363                 return;
 364             }
 365             // VPNInterface got removed, proceed with Add
 366             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 367             VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/);
 368             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, interfaceName,
 369                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 370             if (interfaceManager.isExternalInterface(interfaceName)) {
 371                 processExternalVpnInterface(vpnInterface, vpnId, dpId, lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 372             }
 373         } else {
 374             // Interface is retained in the DPN, but its Link Up.
 375             // Advertise prefixes again for this interface to BGP
 376             advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
 377                     vpnInterface);
 378         }
 379     }
 380
 381     private void processExternalVpnInterface(VpnInterface vpnInterface, long vpnId, BigInteger dpId, int lportTag,
 382             WriteTransaction writeInvTxn, int addOrRemove) {
 383         Uuid extNetworkId;
 384         try {
 385             // vpn instance of ext-net interface is the network-id
 386             extNetworkId = new Uuid(vpnInterface.getVpnInstanceName());
 387         } catch (IllegalArgumentException e) {
 388             LOG.debug("VPN instance {} is not Uuid", vpnInterface.getVpnInstanceName());
 389             return;
 390         }
 391
 392         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 393         if (routerIds == null || routerIds.isEmpty()) {
 394             LOG.debug("No router is associated with {}", extNetworkId.getValue());
 395             return;
 396         }
 397
 398         LOG.debug("Router-ids {} associated with exernal vpn-interface {}", routerIds, vpnInterface.getName());
 399         for (Uuid routerId : routerIds) {
 400             String routerName = routerId.getValue();
 401             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 402             if (Objects.equals(primarySwitch, dpId)) {
 403                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 404                 if (router != null) {
 405                     vpnManager.setupArpResponderFlowsToExternalNetworkIps(routerName,
 406                             VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 407                             dpId, vpnId, vpnInterface.getName(), lportTag, writeInvTxn, addOrRemove);
 408                 } else {
 409                     LOG.error("No external-router found for router-id {}", routerName);
 410                 }
 411             }
 412         }
 413     }
 414
 415     // TODO Clean up the exception handling
 416     @SuppressWarnings("checkstyle:IllegalCatch")
 417     private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
 418             VpnInterface intf) {
 419
 420         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 421         if (rd == null) {
 422             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 423                     intf.getName(), intf.getVpnInstanceName());
 424             return;
 425         } else {
 426             if (rd.equals(intf.getVpnInstanceName())) {
 427                 LOG.info(
 428                         "advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in "
 429                                 + "internal vpn{} with rd {}",
 430                         intf.getName(), intf.getVpnInstanceName(), rd);
 431
 432                 return;
 433             }
 434         }
 435         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
 436                 intf.getVpnInstanceName(), rd);
 437
 438         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 439         if (nextHopIp == null) {
 440             LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
 441             return;
 442         }
 443
 444         //Read NextHops
 445         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 446         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 447         if (adjacencies.isPresent()) {
 448             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 449
 450             if (!nextHops.isEmpty()) {
 451                 LOG.trace("NextHops are {}", nextHops);
 452                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 453                 long l3vni = vpnInstanceOpData.getL3vni();
 454                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 455                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 456                 for (Adjacency nextHop : nextHops) {
 457                     String gatewayMac = null;
 458                     long label = 0;
 459                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 460                         gatewayMac = getGatewayMacAddressForInterface(vpnInstanceOpData.getVpnInstanceName(),
 461                                 intf.getName(), nextHop.getIpAddress()).get();
 462                     } else {
 463                         label = nextHop.getLabel();
 464                     }
 465                     try {
 466                         LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd,
 467                                 nextHop.getIpAddress(), nextHopIp, label);
 468                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 469                                 encapType, (int)label, l3vni, gatewayMac);
 470                         LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd,
 471                                 nextHop.getIpAddress(), nextHopIp, label);
 472                     } catch (Exception e) {
 473                         LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
 474                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 475                     }
 476                 }
 477             }
 478         }
 479     }
 480
 481     // TODO Clean up the exception handling
 482     @SuppressWarnings("checkstyle:IllegalCatch")
 483     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier,
 484             VpnInterface intf) {
 485         //Read NextHops
 486         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 487         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 488
 489         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 490         if (rd == null) {
 491             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 492                     intf.getName(), intf.getVpnInstanceName());
 493             return;
 494         } else {
 495             if (rd.equals(intf.getVpnInstanceName())) {
 496                 LOG.info(
 497                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 498                                 + "internal vpn{} with rd {}",
 499                         intf.getName(), intf.getVpnInstanceName(), rd);
 500                 return;
 501             }
 502         }
 503         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 504                 intf.getVpnInstanceName(), rd);
 505         if (adjacencies.isPresent()) {
 506             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 507
 508             if (!nextHops.isEmpty()) {
 509                 LOG.trace("NextHops are " + nextHops);
 510                 for (Adjacency nextHop : nextHops) {
 511                     try {
 512                         LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 513                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 514                         LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 515                     } catch (Exception e) {
 516                         LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
 517                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 518                     }
 519                 }
 520             }
 521         }
 522     }
 523
 524     @SuppressWarnings("checkstyle:IllegalCatch")
 525     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 526                                                   String interfaceName, final long vpnId,
 527                                                   WriteTransaction writeConfigTxn,
 528                                                   WriteTransaction writeOperTxn,
 529                                                   final WriteTransaction writeInvTxn,
 530                                                   Interface interfaceState) {
 531         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 532         // Read NextHops
 533         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 534         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 535         if (!adjacencies.isPresent()) {
 536             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null, writeOperTxn);
 537             return;
 538         }
 539
 540         // Get the rd of the vpn instance
 541         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 542         String nextHopIp = null;
 543         try {
 544             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 545         } catch (Exception e) {
 546             LOG.warn("Unable to retrieve enpoint ip address for dpnId {} for vpnInterface {} vpnName {}",
 547                     dpnId, interfaceName, vpnName);
 548         }
 549         List<String> nhList = new ArrayList<>();
 550         if (nextHopIp != null) {
 551             nhList.add(nextHopIp);
 552             LOG.trace("NextHop for interface {} is {}", interfaceName, nhList);
 553         }
 554         Optional<String> gwMac = Optional.absent();
 555         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 556         Long l3vni = vpnInstanceOpData.getL3vni();
 557         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 558         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 559         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 560         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 561         List<Adjacency> value = new ArrayList<>();
 562         LOG.trace("NextHops for interface {} are {}", interfaceName, nextHops);
 563         for (Adjacency nextHop : nextHops) {
 564             String rd = primaryRd;
 565             if (nextHop.isPrimaryAdjacency()) {
 566                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 567                 LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, interfaceName, vpnName);
 568                 writeOperTxn.merge(
 569                     LogicalDatastoreType.OPERATIONAL,
 570                     VpnUtil.getPrefixToInterfaceIdentifier(
 571                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 572                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix), true);
 573                 final Uuid subnetId = nextHop.getSubnetId();
 574                 setupGwMacIfRequired(dpnId, vpnName, interfaceName, vpnId, subnetId,
 575                         writeInvTxn, NwConstants.ADD_FLOW, interfaceState);
 576                 final Optional<String> gatewayIp = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 577                 if (gatewayIp.isPresent()) {
 578                     gwMac = getGatewayMacAddressForInterface(vpnName, interfaceName, gatewayIp.get());
 579                     if (gwMac.isPresent()) {
 580                         addArpResponderFlow(dpnId, lportTag, vpnName, vpnId, interfaceName, subnetId,
 581                                 gwMac.get(), gatewayIp.get(), writeInvTxn);
 582                     } else {
 583                         LOG.warn("Gateway MAC for subnet ID {} could not be obtained, cannot create "
 584                                         + "ARP responder flow for interface name {}, vpnName {}, gwIp {}",
 585                                 interfaceName, vpnName, gatewayIp.get());
 586                     }
 587                 } else {
 588                     LOG.warn("Gateway IP for subnet ID {} could not be obtained, cannot create ARP responder flow "
 589                             + "for interface name {}, vpnName {}", subnetId, interfaceName, vpnName);
 590                 }
 591             } else {
 592                 //Extra route adjacency
 593                 LOG.trace("Adding prefix {} and nextHopList {} as extra-route for vpn", nextHop.getIpAddress(),
 594                     nextHop.getNextHopIpList(), vpnName);
 595                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 596                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 597                 synchronized (vpnPrefixKey.intern()) {
 598                     java.util.Optional<String> rdToAllocate = VpnUtil
 599                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker,
 600                             vpnId, Optional.absent(), prefix, vpnName, dpnId, writeOperTxn);
 601                     if (rdToAllocate.isPresent()) {
 602                         rd = rdToAllocate.get();
 603                         LOG.info("The rd {} is allocated for the extraroute {}", rd, prefix);
 604                         VpnUtil.syncUpdate(
 605                                 dataBroker,
 606                                 LogicalDatastoreType.OPERATIONAL,
 607                                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName,
 608                                         rd, nextHop.getIpAddress()),
 609                                 VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()));
 610                     } else {
 611                         LOG.error("No rds to allocate extraroute {}", prefix);
 612                         return;
 613                     }
 614                 }
 615             }
 616             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 617                     .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 618                     .setRouteOrigin(nextHop.isPrimaryAdjacency() ? RouteOrigin.LOCAL : RouteOrigin.STATIC);
 619             Adjacency operationalAdjacency = null;
 620             try {
 621                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 622             } catch (NullPointerException e) {
 623                 LOG.error(e.getMessage());
 624                 return;
 625             }
 626             value.add(operationalAdjacency);
 627         }
 628
 629         Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 630         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, writeOperTxn);
 631
 632         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 633                 .setGatewayMac(gwMac.isPresent() ? gwMac.get() : null).setInterfaceName(interfaceName)
 634                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 635         for (Adjacency nextHop : aug.getAdjacency()) {
 636             RouteOrigin origin = nextHop.isPrimaryAdjacency() ? RouteOrigin.LOCAL : RouteOrigin.STATIC;
 637             input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 638             registeredPopulator.populateFib(input, writeConfigTxn, writeOperTxn);
 639         }
 640     }
 641
 642     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, Adjacencies aug,
 643         WriteTransaction writeOperTxn) {
 644         VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 645         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 646         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, true);
 647     }
 648
 649     // TODO Clean up the exception handling
 650     @SuppressWarnings("checkstyle:IllegalCatch")
 651     public void updateVpnInterfaceOnTepAdd(VpnInterface vpnInterface,
 652         StateTunnelList stateTunnelList) {
 653
 654         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 655         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 656         Adjacencies adjacencies = vpnInterface.getAugmentation(Adjacencies.class);
 657         List<Adjacency> adjList = (adjacencies != null) ? adjacencies.getAdjacency() : new ArrayList<>();
 658         String prefix = null;
 659         long label = 0;
 660         List<String> nhList = new ArrayList<>();
 661         boolean nextHopAdded = false;
 662         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInterface.getVpnInstanceName());
 663
 664         if (adjList != null) {
 665             List<Adjacency> value = new ArrayList<>();
 666             LOG.trace("AdjacencyList for interface {} is {}", vpnInterface, adjList);
 667             for (Adjacency adj : adjList) {
 668                 String rd = adj.getVrfId();
 669                 rd = (rd != null) ? rd : vpnInterface.getVpnInstanceName();
 670                 prefix = adj.getIpAddress();
 671                 label = adj.getLabel();
 672                 nhList = Collections.singletonList(srcTepIp);
 673                 //Only for primary adjacency update the nexthop ip
 674                 if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
 675                     List<String> nextHopList = adj.getNextHopIpList();
 676                     // Incase nextHopList contains some other TEP IP ,
 677                     // it needs to be updated with new one.
 678                     // Incase nextHopIp was null at the time of VpnInterface creation,
 679                     // it needs to be updated with new one.
 680                     if (nextHopList != null && (!nextHopList.isEmpty())
 681                         && (nextHopList.get(0).equalsIgnoreCase(srcTepIp))) {
 682                             /* everything right already */
 683                     } else {
 684                         /* update the adjacency here itself */
 685                         nextHopAdded = true;
 686                         LOG.trace("NextHopList to be updated {}", nhList);
 687                         // Update the VpnInterface Op DS with new nextHopList
 688                         value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 689                         Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 690                         VpnInterface opInterface =
 691                             new VpnInterfaceBuilder(vpnInterface).setKey(new VpnInterfaceKey(vpnInterface.getName()))
 692                                 .addAugmentation(Adjacencies.class, aug).build();
 693                         InstanceIdentifier<VpnInterface> interfaceId =
 694                             VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 695                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface);
 696                     }
 697                 }
 698
 699                 if (nextHopAdded) {
 700                     LOG.info("Updating label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} rd {}", label,
 701                         srcDpnId, prefix, nhList, vpnId, rd);
 702                     updateLabelMapper(label, nhList);
 703                     Optional<String> gwMacAddr = getGatewayMacAddressForInterface(vpnInterface.getVpnInstanceName(),
 704                             vpnInterface.getName(),prefix);
 705                     // Update the VRF entry with nextHop
 706                     String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnInterface.getVpnInstanceName());
 707                     fibManager.updateFibEntry(dataBroker, primaryRd, prefix, nhList,
 708                         gwMacAddr.isPresent() ? gwMacAddr.get() : null, label, null);
 709
 710                     //Get the list of VPN's importing this route(prefix) .
 711                     // Then update the VRF entry with nhList
 712                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 713                         getVpnsImportingMyRoute(vpnInterface.getVpnInstanceName());
 714                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 715                         String vpnRd = vpn.getVrfId();
 716                         if (vpnRd != null) {
 717                             LOG.debug("Exporting route with rd {} prefix {} nhList {} label {} to VPN {}", vpnRd,
 718                                 prefix, nhList, label, vpn);
 719                             fibManager.updateFibEntry(dataBroker, vpnRd, prefix, nhList,
 720                                 gwMacAddr.isPresent() ? gwMacAddr.get() : null, label, null);
 721                         }
 722                     }
 723                     // Advertise the prefix to BGP only for external vpn
 724                     // since there is a nexthop change.
 725                     try {
 726                         if (!rd.equalsIgnoreCase(vpnInterface.getVpnInstanceName())) {
 727                             bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 728                                     VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, null /*gatewayMacAddress*/);
 729                         }
 730                     } catch (Exception ex) {
 731                         LOG.error("Exception when advertising prefix {} on rd {} as {}", prefix, rd, ex);
 732                     }
 733                 }
 734             }
 735         }
 736     }
 737
 738     // TODO Clean up the exception handling
 739     @SuppressWarnings("checkstyle:IllegalCatch")
 740     public void updateVpnInterfaceOnTepDelete(VpnInterface vpnInterface,
 741         StateTunnelList stateTunnelList) {
 742
 743         Adjacencies adjacencies = vpnInterface.getAugmentation(Adjacencies.class);
 744         List<Adjacency> adjList = (adjacencies != null) ? adjacencies.getAdjacency() : new ArrayList<>();
 745         String prefix = null;
 746         long label = 0;
 747         List<String> nhList = new ArrayList<>();
 748         boolean isNextHopRemoveReqd = false;
 749         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 750         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInterface.getVpnInstanceName());
 751
 752         if (adjList != null) {
 753             List<Adjacency> value = new ArrayList<>();
 754             LOG.trace("AdjacencyList for interface {} is {}", vpnInterface, adjList);
 755             for (Adjacency adj : adjList) {
 756                 String rd = adj.getVrfId();
 757                 rd = (rd != null) ? rd : vpnInterface.getVpnInstanceName();
 758                 prefix = adj.getIpAddress();
 759                 label = adj.getLabel();
 760
 761                 // If TEP is deleted , then only remove the nexthop from
 762                 // primary adjacency.
 763                 if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
 764                     List<String> nextHopList = adj.getNextHopIpList();
 765                     // If nextHopList is already cleaned , no need to modify again
 766                     if ((nextHopList != null) & (!nextHopList.isEmpty())) {
 767                         isNextHopRemoveReqd = true;
 768
 769                         value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 770                         Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 771
 772                         VpnInterface opInterface =
 773                             new VpnInterfaceBuilder(vpnInterface).setKey(new VpnInterfaceKey(vpnInterface.getName()))
 774                                 .addAugmentation(Adjacencies.class, aug).build();
 775                         InstanceIdentifier<VpnInterface> interfaceId =
 776                             VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 777                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface);
 778                     }
 779                 }
 780
 781                 if (isNextHopRemoveReqd) {
 782                     LOG.info("Updating label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} rd {}", label,
 783                         srcDpnId, prefix, nhList, vpnId, rd);
 784                     updateLabelMapper(label, nhList);
 785                     Optional<String> gwMacAddr = getGatewayMacAddressForInterface(vpnInterface.getVpnInstanceName(),
 786                         vpnInterface.getName(),prefix);
 787                     // Update the VRF entry with emtpy nextHop
 788                     String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnInterface.getVpnInstanceName());
 789                     fibManager.updateFibEntry(dataBroker, primaryRd, prefix, new ArrayList<>()/* empty */,
 790                             gwMacAddr.isPresent() ? gwMacAddr.get() : null, label, null);
 791
 792                     //Get the list of VPN's importing this route(prefix) .
 793                     // Then update the VRF entry with nhList
 794                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 795                         getVpnsImportingMyRoute(vpnInterface.getVpnInstanceName());
 796                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 797                         String vpnRd = vpn.getVrfId();
 798                         if (vpnRd != null) {
 799                             LOG.debug("Exporting route with rd {} prefix {} nhList {} label {} to VPN {}", vpnRd,
 800                                 prefix, nhList, label, vpn);
 801                             fibManager.updateFibEntry(dataBroker, vpnRd, prefix, nhList,
 802                                 gwMacAddr.isPresent() ? gwMacAddr.get() : null, label, null);
 803                         }
 804                     }
 805
 806                     // Withdraw prefix from BGP only for external vpn.
 807                     try {
 808                         if (!rd.equalsIgnoreCase(vpnInterface.getVpnInstanceName())) {
 809                             bgpManager.withdrawPrefix(rd, prefix);
 810                         }
 811                     } catch (Exception ex) {
 812                         LOG.error("Exception when withdrawing prefix {} on rd {} as {}", prefix, rd, ex);
 813                     }
 814                 }
 815             }
 816         }
 817     }
 818
 819     //TODO (KIRAN) : Move to L3vpnPopulator.
 820     public List<VpnInstanceOpDataEntry> getVpnsImportingMyRoute(final String vpnName) {
 821         List<VpnInstanceOpDataEntry> vpnsToImportRoute = new ArrayList<>();
 822
 823         final String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 824         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
 825         if (vpnInstanceOpDataEntry == null) {
 826             LOG.debug("Could not retrieve vpn instance op data for {} to check for vpns importing the routes", vpnName);
 827             return vpnsToImportRoute;
 828         }
 829
 830         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
 831             if (input.getVpnInstanceName() == null) {
 832                 LOG.error("Received vpn instance without identity");
 833                 return false;
 834             }
 835             return !input.getVpnInstanceName().equals(vpnName);
 836         };
 837
 838         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
 839             Iterable<String> commonRTs =
 840                 intersection(getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ExportExtcommunity),
 841                     getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 842             return Iterators.size(commonRTs.iterator()) > 0;
 843         };
 844
 845         vpnsToImportRoute = VpnUtil.getAllVpnInstanceOpData(dataBroker)
 846                 .stream()
 847                 .filter(excludeVpn)
 848                 .filter(matchRTs)
 849                 .collect(Collectors.toList());
 850         return vpnsToImportRoute;
 851     }
 852
 853     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
 854         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
 855
 856         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 857         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
 858         if (vpnInstanceOpDataEntry == null) {
 859             LOG.debug("Could not retrieve vpn instance op data for {} to check for vpns exporting the routes", vpnName);
 860             return vpnsToExportRoute;
 861         }
 862
 863         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
 864             if (input.getVpnInstanceName() == null) {
 865                 LOG.error("Received vpn instance without identity");
 866                 return false;
 867             }
 868             return !input.getVpnInstanceName().equals(vpnName);
 869         };
 870
 871         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
 872             Iterable<String> commonRTs =
 873                 intersection(getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
 874                     getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
 875             return Iterators.size(commonRTs.iterator()) > 0;
 876         };
 877
 878         vpnsToExportRoute =
 879                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
 880                         Collectors.toList());
 881         return vpnsToExportRoute;
 882     }
 883
 884     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
 885         Set<T> intersection = new HashSet<>(collection1);
 886         intersection.retainAll(collection2);
 887         return intersection;
 888     }
 889
 890     private List<String> getRts(VpnInstanceOpDataEntry vpnInstance, VpnTarget.VrfRTType rtType) {
 891         String name = vpnInstance.getVpnInstanceName();
 892         List<String> rts = new ArrayList<>();
 893         VpnTargets targets = vpnInstance.getVpnTargets();
 894         if (targets == null) {
 895             LOG.trace("vpn targets not available for {}", name);
 896             return rts;
 897         }
 898         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 899         if (vpnTargets == null) {
 900             LOG.trace("vpnTarget values not available for {}", name);
 901             return rts;
 902         }
 903         for (VpnTarget target : vpnTargets) {
 904             //TODO: Check for RT type is Both
 905             if (target.getVrfRTType().equals(rtType) || target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
 906                 String rtValue = target.getVrfRTValue();
 907                 rts.add(rtValue);
 908             }
 909         }
 910         return rts;
 911     }
 912
 913     // TODO Clean up the exception handling
 914     @SuppressWarnings("checkstyle:IllegalCatch")
 915     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
 916         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
 917         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
 918             String rd = vpn.getVrfId();
 919             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
 920             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 921             if (vrfEntries != null) {
 922                 for (VrfEntry vrfEntry : vrfEntries) {
 923                     try {
 924                         if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
 925                                 RouteOrigin.value(vrfEntry.getOrigin()))) {
 926                             continue;
 927                         }
 928                         String prefix = vrfEntry.getDestPrefix();
 929                         String gwMac = vrfEntry.getGatewayMacAddress();
 930                         vrfEntry.getRoutePaths().stream().forEach(routePath -> {
 931                             String nh = routePath.getNexthopAddress();
 932                             int label = routePath.getLabel().intValue();
 933                             if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
 934                                     vrfEntry.getOrigin()))) {
 935                                 LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} gwmac {} to vpn {}",
 936                                         vpnRd, prefix, nh, label, gwMac, vpn.getVpnInstanceName());
 937                                 fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, null /*macAddress*/, prefix,
 938                                         Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre,
 939                                         (int)label, 0 /*l3vni*/, gwMac, RouteOrigin.SELF_IMPORTED, writeConfigTxn);
 940                             } else {
 941                                 LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {}"
 942                                         + " to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
 943                                 SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
 944                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, (int) label, route, writeConfigTxn);
 945                             }
 946                         });
 947                     } catch (Exception e) {
 948                         LOG.error(
 949                             "Exception occurred while importing route with prefix {} route-path {} from vpn {} "
 950                                 + "to vpn {}",
 951                             vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 952                             vpn.getVpnInstanceName(), vpnName);
 953                     }
 954                 }
 955                 writeConfigTxn.submit();
 956             } else {
 957                 LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getVrfId());
 958             }
 959         }
 960     }
 961
 962     @SuppressWarnings("checkstyle:IllegalCatch")
 963     private void addPrefixToBGP(String rd, String primaryRd, String prefix, List<String> nextHopList,
 964                                 VrfEntry.EncapType encapType, long label, long l3vni, String macAddress,
 965                                 String gwMacAddress, RouteOrigin origin, WriteTransaction writeConfigTxn) {
 966         try {
 967             LOG.info("ADD: Adding Fib entry rd {} primaryRd {} prefix {} nextHop {} label {} gwMac {} l3vni {}",
 968                     rd, primaryRd, prefix, nextHopList, label, gwMacAddress, l3vni);
 969             fibManager.addOrUpdateFibEntry(dataBroker, primaryRd, macAddress, prefix, nextHopList,
 970                     encapType, (int)label, l3vni, gwMacAddress, origin, writeConfigTxn);
 971             LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {} gwMac {} l3vni {}",
 972                     rd, prefix, nextHopList, label, gwMacAddress, l3vni);
 973             // Advertize the prefix to BGP only if nexthop ip is available
 974             if (nextHopList != null && !nextHopList.isEmpty()) {
 975                 bgpManager.advertisePrefix(rd, macAddress, prefix, nextHopList, encapType, (int)label,
 976                         l3vni, gwMacAddress);
 977             } else {
 978                 LOG.warn("NextHopList is null/empty. Hence rd {} prefix {} is not advertised to BGP", rd, prefix);
 979             }
 980         } catch (Exception e) {
 981             LOG.error("Add prefix failed.", e);
 982         }
 983     }
 984
 985     @SuppressWarnings("checkstyle:IllegalCatch")
 986     @Override
 987     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
 988         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface);
 989         LOG.info("VPN Interface remove event - intfName {}" ,vpnInterface.getName());
 990         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
 991
 992         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 993         final String interfaceName = key.getName();
 994         BigInteger dpId = BigInteger.ZERO;
 995
 996         Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 997         if (interfaceState != null) {
 998             try {
 999                 dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1000             } catch (Exception e) {
1001                 LOG.error(
1002                     "Unable to retrieve dpnId from interface operational data store for interface {}. Fetching "
1003                         + "from vpn interface op data store. ",
1004                     interfaceName, e);
1005                 dpId = BigInteger.ZERO;
1006             }
1007
1008             final int ifIndex = interfaceState.getIfIndex();
1009             final BigInteger dpnId = dpId;
1010             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1011             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1012                 () -> {
1013                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1014                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1015                     WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
1016                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1017
1018                     InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1019                     final Optional<VpnInterface> optVpnInterface =
1020                             VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1021                     if (optVpnInterface.isPresent()) {
1022                         VpnInterface vpnOpInterface = optVpnInterface.get();
1023                         processVpnInterfaceDown(dpnId.equals(BigInteger.ZERO) ? vpnOpInterface.getDpnId() : dpnId,
1024                                 interfaceName, ifIndex, false, true, writeConfigTxn, writeOperTxn,
1025                                 writeInvTxn, interfaceState);
1026                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
1027                         try {
1028                             operFuture.get();
1029                         } catch (ExecutionException e) {
1030                             LOG.error("Exception encountered while submitting operational future for remove "
1031                                     + "VpnInterface {}: {}", vpnInterface.getName(), e);
1032                             return null;
1033                         }
1034                         futures.add(writeConfigTxn.submit());
1035                         futures.add(writeInvTxn.submit());
1036                     } else {
1037                         LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event",
1038                                 interfaceName);
1039                     }
1040                     return futures;
1041                 });
1042
1043         } else if (Boolean.TRUE.equals(vpnInterface.isIsRouterInterface())) {
1044             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1045             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
1046                 () -> {
1047                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1048                     deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn);
1049                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1050                     futures.add(writeConfigTxn.submit());
1051                     return futures;
1052                 });
1053         } else {
1054             LOG.warn("Handling removal of VPN interface {} skipped as interfaceState is not available", interfaceName);
1055         }
1056     }
1057
1058     protected void processVpnInterfaceDown(BigInteger dpId,
1059                                            String interfaceName,
1060                                            int lportTag,
1061                                            boolean isInterfaceStateDown,
1062                                            boolean isConfigRemoval,
1063                                            WriteTransaction writeConfigTxn,
1064                                            WriteTransaction writeOperTxn,
1065                                            WriteTransaction writeInvTxn,
1066                                            Interface interfaceState) {
1067         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1068         if (!isInterfaceStateDown) {
1069             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1070             if (vpnInterface == null) {
1071                 LOG.info(
1072                     "Unable to process delete/down for interface {} as it is not available in operational data store",
1073                     interfaceName);
1074                 return;
1075             } else {
1076                 final String vpnName = vpnInterface.getVpnInstanceName();
1077                 final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1078                 if (!vpnInterface.isScheduledForRemove()) {
1079                     VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1080                             writeOperTxn);
1081                     removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnInterface.getVpnInstanceName(),
1082                             vpnId, writeConfigTxn, writeInvTxn, interfaceState);
1083                     if (interfaceManager.isExternalInterface(interfaceName)) {
1084                         processExternalVpnInterface(vpnInterface, vpnId, dpId, lportTag, writeInvTxn,
1085                                 NwConstants.DEL_FLOW);
1086                     }
1087                     LOG.info("Unbinding vpn service from interface {} ", interfaceName);
1088                     VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, isConfigRemoval);
1089
1090                 } else {
1091                     LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ",
1092                             interfaceName);
1093                     return;
1094                 }
1095             }
1096         } else {
1097             // Interface is retained in the DPN, but its Link Down.
1098             // Only withdraw the prefixes for this interface from BGP
1099             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1100             if (vpnInterface == null) {
1101                 LOG.info(
1102                     "Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in "
1103                         + "operational data store",
1104                     interfaceName);
1105                 return;
1106             } else {
1107                 withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
1108             }
1109         }
1110     }
1111
1112     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1113                                           final String vpnName, final long vpnId, WriteTransaction writeConfigTxn,
1114                                           final WriteTransaction writeInvTxn, Interface interfaceState) {
1115         //Read NextHops
1116         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1117         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1118         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1119
1120         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1121         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", interfaceName,
1122             vpnName, rd);
1123         if (adjacencies.isPresent()) {
1124             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1125
1126             if (!nextHops.isEmpty()) {
1127                 LOG.trace("NextHops are " + nextHops);
1128                 for (Adjacency nextHop : nextHops) {
1129                     List<String> nhList = new ArrayList<>();
1130                     if (!nextHop.isPrimaryAdjacency()) {
1131                         // This is either an extra-route (or) a learned IP via subnet-route
1132                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1133                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1134                             LOG.warn("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
1135                                 rd, nextHop.getIpAddress());
1136                         } else {
1137                             nhList = Collections.singletonList(nextHopIp);
1138                         }
1139                     } else {
1140                         // This is a primary adjacency
1141                         nhList = nextHop.getNextHopIpList();
1142                         final Uuid subnetId = nextHop.getSubnetId();
1143                         setupGwMacIfRequired(dpnId, vpnName, interfaceName, vpnId, subnetId,
1144                                 writeInvTxn, NwConstants.DEL_FLOW, interfaceState);
1145                         removeArpResponderFlow(dpnId, lportTag, subnetId, writeInvTxn);
1146                     }
1147
1148                     if (!nhList.isEmpty()) {
1149                         if (rd.equals(vpnName)) {
1150                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1151                             //remove from FIB directly
1152                             for (String nh : nhList) {
1153                                 fibManager.removeOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), nh,
1154                                     writeConfigTxn);
1155                             }
1156                         } else {
1157                             List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1158                             for (String nh : nhList) {
1159                                 //IRT: remove routes from other vpns importing it
1160                                 removePrefixFromBGP(rd, vpnName, nextHop.getIpAddress(), nh, writeConfigTxn);
1161                                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1162                                     String vpnRd = vpn.getVrfId();
1163                                     if (vpnRd != null) {
1164                                         LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd,
1165                                             nextHop.getIpAddress(), vpn.getVpnInstanceName());
1166                                         fibManager.removeOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), nh,
1167                                             writeConfigTxn);
1168                                     }
1169                                 }
1170                             }
1171                         }
1172                     } else {
1173                         fibManager.removeFibEntry(dataBroker, rd, nextHop.getIpAddress(), writeConfigTxn);
1174                     }
1175
1176                     String ip = nextHop.getIpAddress().split("/")[0];
1177                     LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1178                     if (vpnVipToPort != null) {
1179                         LOG.trace(
1180                                 "VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
1181                                 vpnVipToPort.getPortName(), ip);
1182                         VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip);
1183                     }
1184                 }
1185             }
1186         }
1187     }
1188
1189     private  void addArpResponderFlow(final BigInteger dpId, final int lportTag, final String vpnName,
1190                                       final long vpnId, final String ifName, final Uuid subnetId,
1191                                       final String subnetGwMac, final String gwIp, final WriteTransaction writeInvTxn) {
1192         LOG.trace("Creating the ARP Responder flow for VPN Interface {}",ifName);
1193         final String flowId = ArpResponderUtil.getFlowID(lportTag, gwIp);
1194         List<Action> actions = ArpResponderUtil.getActions(ifaceMgrRpcService, ifName, gwIp, subnetGwMac);
1195         ArpResponderUtil.installFlow(mdsalManager, writeInvTxn, dpId, flowId, flowId,
1196                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, ArpResponderUtil.generateCookie(lportTag, gwIp),
1197                 ArpResponderUtil.getMatchCriteria(lportTag, vpnId, gwIp),
1198                 Collections.singletonList(MDSALUtil.buildApplyActionsInstruction(actions)));
1199         LOG.trace("Installed the ARP Responder flow for VPN Interface {}", ifName);
1200     }
1201
1202     private Optional<String> getGatewayMacAddressForInterface(String vpnName, String ifName, String ipAddress) {
1203         Optional<String> routerGwMac = Optional.absent();
1204         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1205         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1206         // else use connected interface
1207         routerGwMac = Optional.of((gwPort != null && gwPort.isSubnetIp())
1208                 ? gwPort.getMacAddress() : InterfaceUtils.getMacAddressForInterface(dataBroker, ifName).get());
1209         return routerGwMac;
1210     }
1211
1212     private void removeArpResponderFlow(final BigInteger dpId, final int lportTag, final Uuid subnetUuid,
1213                                         final WriteTransaction writeInvTxn) {
1214         final Optional<String> gwIp = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetUuid);
1215         if (gwIp.isPresent()) {
1216             LOG.trace("VPNInterface adjacency Gsteway IP {} for ARP Responder removal", gwIp.get());
1217             final String flowId = ArpResponderUtil.getFlowID(lportTag, gwIp.get());
1218             ArpResponderUtil.removeFlow(mdsalManager, writeInvTxn, dpId, flowId);
1219         }
1220     }
1221
1222     private void setupGwMacIfRequired(BigInteger dpId, String vpnInstanceName, final String vpnInterfaceName,
1223                                       long vpnId, final Uuid subnetUuid, WriteTransaction writeInvTxn,
1224                                       int addOrRemove, Interface interfaceState) {
1225         // check for router is present for the given vpn interface, if present return it immediately and
1226         // do not need to proceed with adding/removing interface mac as l3_gwmac_table flow entry
1227         Optional<VpnPortipToPort> routerInterfaceOptional = VpnUtil.getRouterInterfaceForVpnInterface(dataBroker,
1228                 vpnInterfaceName, vpnInstanceName, subnetUuid);
1229         if (routerInterfaceOptional.isPresent() && routerInterfaceOptional.get().getMacAddress() != null) {
1230             return;
1231         }
1232         VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpId, vpnInterfaceName,
1233                 vpnId, writeInvTxn, addOrRemove, interfaceState);
1234     }
1235
1236     // TODO Clean up the exception handling
1237     @SuppressWarnings("checkstyle:IllegalCatch")
1238     private void removePrefixFromBGP(String rd, String vpnName, String prefix, String nextHop,
1239                                      WriteTransaction writeConfigTxn) {
1240         try {
1241             LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
1242             fibManager.removeOrUpdateFibEntry(dataBroker, rd, prefix, nextHop, writeConfigTxn);
1243             if (rd != null && !rd.equalsIgnoreCase(vpnName)) {
1244                 bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1245             }
1246             LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
1247         } catch (Exception e) {
1248             LOG.error("Delete prefix failed", e);
1249         }
1250     }
1251
1252     @Override
1253     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1254         final VpnInterface update) {
1255         LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
1256         LOG.info("VPN Interface update event - intfName {}", update.getName());
1257         final String vpnInterfaceName = update.getName();
1258         final String oldVpnName = original.getVpnInstanceName();
1259         final String newVpnName = update.getVpnInstanceName();
1260         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1261         final UpdateData updateData = new UpdateData(identifier, original, update);
1262         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1263         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency()
1264             != null) ? origAdjs.getAdjacency() : new ArrayList<>();
1265         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1266         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency()
1267             != null) ? updateAdjs.getAdjacency() : new ArrayList<>();
1268
1269         //handles switching between <internal VPN - external VPN>
1270         if (oldVpnName != null && !oldVpnName.equals(newVpnName)) {
1271             vpnInterfacesUpdateQueue.add(updateData);
1272             LOG.trace("UpdateData on VPNInterface {} update upon VPN swap added to update queue",
1273                 updateData.getOriginal().getName());
1274             return;
1275         }
1276         final DataStoreJobCoordinator vpnInfAdjUpdateDataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1277         vpnInfAdjUpdateDataStoreCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName,
1278             () -> {
1279                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1280                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1281                 //handle both addition and removal of adjacencies
1282                 //currently, new adjacency may be an extra route
1283                 if (!oldAdjs.equals(newAdjs)) {
1284                     for (Adjacency adj : newAdjs) {
1285                         if (oldAdjs.contains(adj)) {
1286                             oldAdjs.remove(adj);
1287                         } else {
1288                             // add new adjacency - right now only extra route will hit this path
1289                             addNewAdjToVpnInterface(identifier, adj, dpnId, writeOperTxn, writeConfigTxn);
1290                         }
1291                     }
1292                     for (Adjacency adj : oldAdjs) {
1293                         delAdjFromVpnInterface(identifier, adj, dpnId, writeOperTxn, writeConfigTxn);
1294                     }
1295                 }
1296                 ListenableFuture<Void> operFuture = writeOperTxn.submit();
1297                 try {
1298                     operFuture.get();
1299                 } catch (ExecutionException e) {
1300                     LOG.error("Exception encountered while submitting operational future for update VpnInterface {}: "
1301                             + "{}", vpnInterfaceName, e);
1302                     return null;
1303                 }
1304                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1305                 futures.add(writeConfigTxn.submit());
1306                 return futures;
1307             });
1308     }
1309
1310     class VpnInterfaceUpdateTimerTask extends TimerTask {
1311
1312         @Override
1313         public void run() {
1314             List<UpdateData> processQueue = new ArrayList<>();
1315             List<VpnInterface> vpnInterfaceList = new ArrayList<>();
1316             vpnInterfacesUpdateQueue.drainTo(processQueue);
1317             for (UpdateData updData : processQueue) {
1318                 remove(updData.getIdentifier(), updData.getOriginal());
1319                 LOG.trace("Processed Remove for update on VPNInterface {} upon VPN swap",
1320                     updData.getOriginal().getName());
1321                 vpnInterfaceList.add(updData.getOriginal());
1322             }
1323
1324             /* Decide the max-wait time based on number of VpnInterfaces.
1325             *  max-wait-time is num-of-interface * 4seconds (random choice).
1326             *  Every 2sec poll VpnToDpnList. If VpnInterface is removed ,
1327             *  remove it from vpnInterfaceList.
1328             */
1329
1330             int maxWaitTime =
1331                 vpnInterfaceList.size() * (int) (VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS / 1000);
1332             int waitTime = 2;
1333             Iterator<VpnInterface> vpnInterfaceIterator = vpnInterfaceList.iterator();
1334             VpnInterface vpnInterface;
1335             while (waitTime < maxWaitTime) {
1336                 try {
1337                     Thread.sleep(2000); // sleep for 2sec
1338                 } catch (InterruptedException e) {
1339                     // Ignored
1340                 }
1341
1342                 while (vpnInterfaceIterator.hasNext()) {
1343                     vpnInterface = vpnInterfaceIterator.next();
1344                     if (!VpnUtil.isVpnIntfPresentInVpnToDpnList(dataBroker, vpnInterface)) {
1345                         vpnInterfaceIterator.remove();
1346                     }
1347                 }
1348                 if (vpnInterfaceList.size() == 0) {
1349                     LOG.trace("All VpnInterfaces are successfully removed from OLD VPN after time {}", waitTime);
1350                     break;
1351                 }
1352                 waitTime += 2; //Increment linearly by 2sec.
1353             }
1354
1355             if (vpnInterfaceList.size() > 0) {
1356                 LOG.error("VpnInterfacesList {} not removed from old Vpn even after waiting {}", vpnInterfaceList,
1357                     waitTime);
1358             }
1359
1360             for (UpdateData updData : processQueue) {
1361                 if (vpnInterfaceList.contains(updData.getOriginal())) {
1362                     LOG.warn("Failed to swap VpnInterfaces {} to target VPN {}", updData.getOriginal(),
1363                         updData.getUpdate().getVpnInstanceName());
1364                     continue;
1365                 }
1366                 final Adjacencies origAdjs = updData.getOriginal().getAugmentation(Adjacencies.class);
1367                 final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1368                     ? origAdjs.getAdjacency() : new ArrayList<>();
1369                 final Adjacencies updateAdjs = updData.getUpdate().getAugmentation(Adjacencies.class);
1370                 final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1371                     ? updateAdjs.getAdjacency() : new ArrayList<>();
1372                 addVpnInterface(updData.getIdentifier(), updData.getUpdate(), oldAdjs, newAdjs);
1373                 LOG.trace("Processed Add for update on VPNInterface {} upon VPN swap",
1374                     updData.getUpdate().getName());
1375             }
1376         }
1377     }
1378
1379     private String getErrorText(Collection<RpcError> errors) {
1380         StringBuilder errorText = new StringBuilder();
1381         for (RpcError error : errors) {
1382             errorText.append(",").append(error.getErrorType()).append("-")
1383                 .append(error.getMessage());
1384         }
1385         return errorText.toString();
1386     }
1387
1388     //TODO (KIRAN) : Move to implemetation specific L3vpnOverMplsGrePopulator
1389     public void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1390                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd,
1391                                   WriteTransaction writeOperTxn) {
1392         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1393         Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
1394         Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
1395         Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
1396         if (!isSubnetRoute) {
1397             // NextHop must be present for non-subnetroute entries
1398             Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
1399         }
1400         LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}",
1401             label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
1402         if (dpnId != null) {
1403             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1404                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) label)).build();
1405             LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1406             lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(
1407                 vpnId)
1408                 .setIsSubnetRoute(isSubnetRoute);
1409             if (elanTag != null) {
1410                 lriBuilder.setElanTag(elanTag);
1411             }
1412             if (vpnInterfaceName != null) {
1413                 lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1414             }
1415             lriBuilder.setParentVpnRd(rd);
1416             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
1417             if (vpnInstanceOpDataEntry != null) {
1418                 List<String> vpnInstanceNames = Collections.singletonList(vpnInstanceOpDataEntry.getVpnInstanceName());
1419                 lriBuilder.setVpnInstanceList(vpnInstanceNames);
1420             }
1421             LabelRouteInfo lri = lriBuilder.build();
1422             LOG.trace("Adding route info to label map: {}", lri);
1423             if (writeOperTxn != null) {
1424                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, lriIid, lri, true);
1425             } else {
1426                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
1427             }
1428         } else {
1429             LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
1430         }
1431     }
1432
1433     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1434         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1435
1436         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1437             .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) label)).build();
1438         Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1439         if (opResult.isPresent()) {
1440             LabelRouteInfo labelRouteInfo =
1441                 new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1442             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1443         }
1444     }
1445
1446     public void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
1447         long elantag, BigInteger dpnId, WriteTransaction writeTxn) {
1448         SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
1449         RouteOrigin origin = RouteOrigin.CONNECTED; // Only case when a route is considered as directly connected
1450         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin)
1451                 .addAugmentation(SubnetRoute.class, route).build();
1452
1453         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
1454
1455         //TODO: What should be parentVpnId? Get it from RD?
1456         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1457         addToLabelMapper((long) label, dpnId, prefix, Collections.singletonList(nextHop), vpnId, null, elantag, true,
1458                 rd, null);
1459         InstanceIdentifier<VrfEntry> vrfEntryId =
1460             InstanceIdentifier.builder(FibEntries.class)
1461                 .child(VrfTables.class, new VrfTablesKey(rd))
1462                 .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
1463         Optional<VrfEntry> entry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1464
1465         if (!entry.isPresent()) {
1466             List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1467
1468             InstanceIdentifierBuilder<VrfTables> idBuilder =
1469                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1470             InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1471
1472             VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1473
1474             if (writeTxn != null) {
1475                 writeTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1476             } else {
1477                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1478             }
1479         } else { // Found in MDSAL database
1480             if (writeTxn != null) {
1481                 writeTxn.put(LogicalDatastoreType.CONFIGURATION, vrfEntryId, vrfEntry, true);
1482             } else {
1483                 VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId, vrfEntry);
1484             }
1485             LOG.debug("Updated vrfEntry for {} nexthop {} label {}", prefix, nextHop, label);
1486         }
1487
1488         List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1489         if (vpnsToImportRoute.size() > 0) {
1490             VrfEntry importingVrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, RouteOrigin.SELF_IMPORTED)
1491                     .addAugmentation(SubnetRoute.class, route).build();
1492             List<VrfEntry> importingVrfEntryList = Collections.singletonList(importingVrfEntry);
1493             for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
1494                 LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop,
1495                     label, vpnInstance.getVpnInstanceName());
1496                 String importingRd = vpnInstance.getVrfId();
1497                 InstanceIdentifier<VrfTables> importingVrfTableId =
1498                     InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class,
1499                         new VrfTablesKey(importingRd)).build();
1500                 VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(
1501                     importingVrfEntryList).build();
1502                 if (writeTxn != null) {
1503                     writeTxn.merge(LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable, true);
1504                 } else {
1505                     VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId,
1506                         importingVrfTable);
1507                 }
1508             }
1509         }
1510     }
1511
1512     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1513         SubnetRoute route, WriteTransaction writeConfigTxn) {
1514
1515         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1516         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin)
1517                 .addAugmentation(SubnetRoute.class, route).build();
1518         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label,
1519                 route.getElantag());
1520         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1521         InstanceIdentifierBuilder<VrfTables> idBuilder =
1522             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1523         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1524         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1525         if (writeConfigTxn != null) {
1526             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1527         } else {
1528             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1529         }
1530     }
1531
1532     public void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName) {
1533         fibManager.removeFibEntry(dataBroker, rd, prefix, null);
1534         List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1535         for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
1536             String importingRd = vpnInstance.getVrfId();
1537             LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix,
1538                 vpnInstance.getVpnInstanceName());
1539             fibManager.removeFibEntry(dataBroker, importingRd, prefix, null);
1540         }
1541     }
1542
1543     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId,
1544         WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1545
1546         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1547
1548         if (optVpnInterface.isPresent()) {
1549             VpnInterface currVpnIntf = optVpnInterface.get();
1550             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1551             String vpnName = currVpnIntf.getVpnInstanceName();
1552             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
1553             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1554             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, adjPath);
1555             long label =
1556                 VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1557                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1558             if (label == 0) {
1559                 LOG.error(
1560                     "Unable to fetch label from Id Manager. Bailing out of adding new adjacency {} to vpn interface "
1561                         + "{} for vpn {}",
1562                     adj.getIpAddress(), currVpnIntf.getName(), currVpnIntf.getVpnInstanceName());
1563                 return;
1564             }
1565             List<Adjacency> adjacencies;
1566             if (optAdjacencies.isPresent()) {
1567                 adjacencies = optAdjacencies.get().getAdjacency();
1568             } else {
1569                 //This code will not be hit since VM adjacency will always be there
1570                 adjacencies = new ArrayList<>();
1571             }
1572             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1573             AdjacencyBuilder adjBuilder = new AdjacencyBuilder(adj).setLabel(label)
1574                     .setNextHopIpList(adj.getNextHopIpList()).setIpAddress(prefix).setKey(new AdjacencyKey(prefix));
1575             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) {
1576                 RouteOrigin origin = adj.isPrimaryAdjacency() ? RouteOrigin.LOCAL : RouteOrigin.STATIC;
1577                 String nh = adj.getNextHopIpList().get(0);
1578                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1579                 synchronized (vpnPrefixKey.intern()) {
1580                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1581                             dataBroker, vpnId, Optional.absent(), prefix, vpnName, dpnId,writeOperTxn);
1582                     if (rdToAllocate.isPresent()) {
1583                         adjBuilder.setVrfId(rdToAllocate.get());
1584                         addExtraRoute(vpnName, adj.getIpAddress(), nh,rdToAllocate.get(),
1585                                 currVpnIntf.getVpnInstanceName(), (int) label,
1586                                 origin, currVpnIntf.getName(), writeConfigTxn);
1587                     } else {
1588                         LOG.error("No rds to allocate extraroute {}", prefix);
1589                         return;
1590                     }
1591                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1592                     vpnsToImportRoute.stream().forEach(vpn -> {
1593                         java.util.Optional.ofNullable(vpn.getVrfId()).ifPresent(vpnRd -> {
1594                             java.util.Optional.ofNullable(VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1595                                     dataBroker, vpn.getVpnId(), Optional.fromNullable(vpnId), prefix,
1596                                     VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), dpnId,
1597                                     writeOperTxn)).ifPresent(rdsToAllocate -> {
1598                                         addExtraRoute(VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1599                                                 adj.getIpAddress(), nh, rdsToAllocate.get(),
1600                                                 currVpnIntf.getVpnInstanceName(), (int) label,
1601                                                 RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), writeConfigTxn);
1602                                     });
1603                         });
1604                     });
1605                 }
1606             } else {
1607                 adjBuilder.setVrfId(primaryRd);
1608             }
1609             adjacencies.add(adjBuilder.build());
1610             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1611             VpnInterface newVpnIntf =
1612                 VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId,
1613                     currVpnIntf.isScheduledForRemove());
1614
1615             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1616         }
1617     }
1618
1619     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId,
1620             WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1621         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1622
1623         if (optVpnInterface.isPresent()) {
1624             VpnInterface currVpnIntf = optVpnInterface.get();
1625
1626             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1627             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1628             if (optAdjacencies.isPresent()) {
1629                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1630
1631                 if (!adjacencies.isEmpty()) {
1632                     String rd = VpnUtil.getVpnRd(dataBroker, currVpnIntf.getVpnInstanceName());
1633                     LOG.trace("Adjacencies are " + adjacencies);
1634                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1635                     while (adjIt.hasNext()) {
1636                         Adjacency adjElem = adjIt.next();
1637                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1638                             String usedRd = adjElem.getVrfId();
1639                             adjIt.remove();
1640
1641                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1642                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1643                                     currVpnIntf.getVpnInstanceName(),
1644                                     aug, dpnId, currVpnIntf.isScheduledForRemove());
1645
1646                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1647                             if (adj.getNextHopIpList() != null) {
1648                                 for (String nh : adj.getNextHopIpList()) {
1649                                     delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
1650                                             currVpnIntf.getName(), writeConfigTxn);
1651                                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1652                                             getVpnsImportingMyRoute(currVpnIntf.getVpnInstanceName());
1653                                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1654                                         java.util.Optional.ofNullable(vpn.getVrfId()).ifPresent(vpnRd -> {
1655                                             delExtraRoute(adj.getIpAddress(), nh, vpnRd,
1656                                                     currVpnIntf.getVpnInstanceName(),
1657                                                     currVpnIntf.getName(), writeConfigTxn);
1658                                         });
1659                                     }
1660                                 }
1661                             }
1662                             break;
1663                         }
1664
1665                     }
1666                 }
1667             }
1668         }
1669     }
1670
1671     protected void addExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
1672                                  int label, RouteOrigin origin, String intfName,
1673                                  WriteTransaction writeConfigTxn) {
1674
1675         Boolean writeConfigTxnPresent = true;
1676         if (writeConfigTxn == null) {
1677             writeConfigTxnPresent = false;
1678             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1679         }
1680
1681         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1682         VpnUtil.syncUpdate(
1683                 dataBroker,
1684                 LogicalDatastoreType.OPERATIONAL,
1685                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, (rd != null) ? rd : routerID,
1686                         destination),
1687                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
1688
1689         BigInteger dpnId = null;
1690         if (intfName != null && !intfName.isEmpty()) {
1691             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1692             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1693             if (nextHopIp == null || nextHopIp.isEmpty()) {
1694                 LOG.error(
1695                     "NextHop for interface {} is null / empty. Failed advertising extra route for rd {} prefix {}",
1696                     intfName, rd, destination);
1697                 return;
1698             }
1699             nextHop = nextHopIp;
1700         }
1701         List<String> nextHopIpList = Collections.singletonList(nextHop);
1702         if (rd != null) {
1703             /* Label mapper is required only for BGP VPN and not for Internal VPN */
1704             addToLabelMapper((long) label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(dataBroker, routerID),
1705                 intfName, null, false, rd, null);
1706         }
1707
1708         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
1709         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1710         // first place.
1711         Optional<InterVpnLinkDataComposite> optVpnLink = InterVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
1712         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
1713             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
1714             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1715             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1716             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1717             // This is like leaking one of the Vpn2 routes towards Vpn1
1718             boolean nexthopIsVpn2 = interVpnLink.getSecondEndpointVpnUuid().get().equals(nextHop);
1719             String srcVpnUuid = nexthopIsVpn2 ? interVpnLink.getSecondEndpointVpnUuid().get()
1720                                               : interVpnLink.getFirstEndpointVpnUuid().get();
1721             String dstVpnUuid = interVpnLink.getOtherVpnName(srcVpnUuid);
1722             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
1723             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1724                                                 VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1725             if (newLabel == 0) {
1726                 LOG.error("Unable to fetch label from Id Manager. Bailing out of adding intervpnlink route "
1727                           + "for destination {}", destination);
1728                 return;
1729             }
1730             InterVpnLinkUtil.leakRoute(dataBroker, bgpManager, interVpnLink.getInterVpnLinkConfig(),
1731                                        srcVpnUuid, dstVpnUuid, destination, newLabel);
1732         } else {
1733             if (rd != null) {
1734                 addPrefixToBGP(rd, VpnUtil.getPrimaryRd(dataBroker, vpnName), destination, nextHopIpList,
1735                         VrfEntry.EncapType.Mplsgre, label, 0 /*l3vni*/, null /*macAddress*/,
1736                         null /*gatewayMacAddress*/, origin, writeConfigTxn);
1737             } else {
1738                 // ### add FIB route directly
1739                 // ### add FIB route directly
1740                 fibManager.addOrUpdateFibEntry(dataBroker, routerID, null /*macAddress*/, destination,
1741                         Collections.singletonList(nextHop), VrfEntry.EncapType.Mplsgre, label, 0 /*l3vni*/,
1742                         null /*gatewayMacAddress*/, origin, writeConfigTxn);
1743             }
1744         }
1745         if (!writeConfigTxnPresent) {
1746             writeConfigTxn.submit();
1747         }
1748     }
1749
1750     protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName,
1751         WriteTransaction writeConfigTxn) {
1752         Boolean writeConfigTxnPresent = true;
1753         if (writeConfigTxn == null) {
1754             writeConfigTxnPresent = false;
1755             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1756         }
1757         if (intfName != null && !intfName.isEmpty()) {
1758             BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1759             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1760             if (nextHopIp == null || nextHopIp.isEmpty()) {
1761                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1762                     intfName, destination);
1763             }
1764             nextHop = nextHopIp;
1765         }
1766         if (rd != null) {
1767             String vpnName = VpnUtil.getVpnNameFromRd(dataBroker, rd);
1768             removePrefixFromBGP(rd, vpnName, destination, nextHop, writeConfigTxn);
1769         } else {
1770             // ### add FIB route directly
1771             fibManager.removeOrUpdateFibEntry(dataBroker, routerID, destination, nextHop, writeConfigTxn);
1772         }
1773         if (!writeConfigTxnPresent) {
1774             writeConfigTxn.submit();
1775         }
1776     }
1777
1778
1779     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1780         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1781             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1782             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1783     }
1784
1785     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1786         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1787             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1788     }
1789
1790     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1791         WriteTransaction writeOperTxn) {
1792         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1793         if (dpId.equals(BigInteger.ZERO)) {
1794             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model",
1795                 vpnInterfaceName, routerName);
1796             return;
1797         }
1798         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1799
1800         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1801             .OPERATIONAL, routerDpnListIdentifier);
1802         RouterInterfaces routerInterface =
1803             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1804                 vpnInterfaceName).build();
1805         if (optionalRouterDpnList.isPresent()) {
1806             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1807                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1808         } else {
1809             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1810             builder.setRouterId(routerName);
1811             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1812             List<RouterInterfaces> routerInterfaces = new ArrayList<>();
1813             routerInterfaces.add(routerInterface);
1814             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
1815             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1816                 getRouterId(routerName),
1817                 builder.build(), true);
1818         }
1819     }
1820
1821     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1822         WriteTransaction writeOperTxn) {
1823         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1824         if (dpId.equals(BigInteger.ZERO)) {
1825             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model",
1826                 vpnInterfaceName, routerName);
1827             return;
1828         }
1829         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1830         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1831             .OPERATIONAL, routerDpnListIdentifier);
1832         if (optionalRouterDpnList.isPresent()) {
1833             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1834             RouterInterfaces routerInterface =
1835                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1836                     vpnInterfaceName).build();
1837
1838             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1839                 if (routerInterfaces.isEmpty()) {
1840                     if (writeOperTxn != null) {
1841                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1842                     } else {
1843                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1844                     }
1845                 } else {
1846                     if (writeOperTxn != null) {
1847                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1848                             RouterInterfaces.class,
1849                             new RouterInterfacesKey(vpnInterfaceName)));
1850                     } else {
1851                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1852                             routerDpnListIdentifier.child(
1853                                 RouterInterfaces.class,
1854                                 new RouterInterfacesKey(vpnInterfaceName)));
1855                     }
1856                 }
1857             }
1858         }
1859     }
1860
1861     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1862         WriteTransaction writeOperTxn) {
1863         if (dpId.equals(BigInteger.ZERO)) {
1864             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model",
1865                 vpnInterfaceName, routerName);
1866             return;
1867         }
1868         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1869         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1870             .OPERATIONAL, routerDpnListIdentifier);
1871         if (optionalRouterDpnList.isPresent()) {
1872             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1873             RouterInterfaces routerInterface =
1874                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1875                     vpnInterfaceName).build();
1876             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1877                 if (routerInterfaces.isEmpty()) {
1878                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1879                 } else {
1880                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1881                         RouterInterfaces.class,
1882                         new RouterInterfacesKey(vpnInterfaceName)));
1883                 }
1884             }
1885         }
1886     }
1887
1888     protected void createFibEntryForRouterInterface(VpnInterface vpnInterface, String interfaceName,
1889                                                     WriteTransaction writeConfigTxn) {
1890         if (vpnInterface == null) {
1891             return;
1892         }
1893         String vpnName = vpnInterface.getVpnInstanceName();
1894         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
1895         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1896         if (adjs == null) {
1897             LOG.info("VPN Interface {} of router addition failed as adjacencies for "
1898                 + "this vpn interface could not be obtained", interfaceName);
1899             return;
1900         }
1901         for (Adjacency adj : adjs) {
1902             if (adj.isPrimaryAdjacency()) {
1903                 String primaryInterfaceIp = adj.getIpAddress();
1904                 String macAddress = adj.getMacAddress();
1905                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1906
1907                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1908                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1909
1910                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1911                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1912                 fibManager.addFibEntryForRouterInterface(dataBroker, primaryRd, prefix,
1913                         routerInt, label, writeConfigTxn);
1914                 return;
1915             }
1916         }
1917         LOG.trace("VPN Interface {} of router addition failed as primary adjacency for"
1918             + " this vpn interface could not be obtained", interfaceName);
1919     }
1920
1921     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface, WriteTransaction writeConfigTxn) {
1922         List<Adjacency> adjsList = new ArrayList<>();
1923         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
1924         if (adjs != null) {
1925             adjsList = adjs.getAdjacency();
1926             for (Adjacency adj : adjsList) {
1927                 if (adj.isPrimaryAdjacency()) {
1928                     String primaryInterfaceIp = adj.getIpAddress();
1929                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1930                     String rd = VpnUtil.getVpnRd(dataBroker, vpnInterface.getVpnInstanceName());
1931                     fibManager.removeFibEntry(dataBroker, rd, prefix, writeConfigTxn);
1932                     return;
1933                 }
1934             }
1935         }
1936     }
1937
1938     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData) {
1939         addVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, null, null);
1940     }
1941
1942     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1943         LOG.info("Saving unhandled vpn interface {} in vpn instance {}",
1944                  vpnInterface.getName(), vpnInterface.getVpnInstanceName());
1945
1946         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
1947                 .get(vpnInterface.getVpnInstanceName());
1948         if (vpnInterfaces == null) {
1949             vpnInterfaces = new ConcurrentLinkedQueue<>();
1950         }
1951         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
1952         unprocessedVpnInterfaces.put(vpnInterface.getVpnInstanceName(), vpnInterfaces);
1953     }
1954
1955     private boolean isVpnInstanceReady(String vpnInstanceName) {
1956         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
1957         if (vpnRd == null) {
1958             return false;
1959         }
1960         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1961
1962         return (vpnInstanceOpDataEntry != null);
1963     }
1964
1965     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
1966         synchronized (vpnInstanceName.intern()) {
1967             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
1968                     unprocessedVpnInterfaces.get(vpnInstanceName);
1969             if (vpnInterfaces != null) {
1970                 while (!vpnInterfaces.isEmpty()) {
1971                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
1972                     LOG.info("Handle saved vpn interface {} in vpn instance {}",
1973                              savedInterface.vpnInterface.getName(), vpnInstanceName);
1974
1975                     if (hasVpnInstanceCreatedSuccessfully) {
1976                         processSavedInterface(savedInterface);
1977                     } else {
1978                         LOG.error("Cannot process vpn interface {} in vpn instance {}",
1979                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
1980                     }
1981                 }
1982             }
1983         }
1984     }
1985
1986     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
1987             VpnInterface vpnInterface) {
1988         synchronized (vpnInterface.getVpnInstanceName().intern()) {
1989             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
1990                     unprocessedVpnInterfaces.get(vpnInterface.getVpnInstanceName());
1991             if (vpnInterfaces != null) {
1992                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
1993                     LOG.info("Removed vpn interface {} in vpn instance {} from unprocessed list",
1994                               vpnInterface.getName(), vpnInterface.getVpnInstanceName());
1995                 }
1996             }
1997         }
1998     }
1999
2000     public void vpnInstanceIsReady(String vpnInstanceName) {
2001         processSavedInterfaces(vpnInstanceName, true);
2002     }
2003
2004     public void vpnInstanceFailed(String vpnInstanceName) {
2005         processSavedInterfaces(vpnInstanceName, false);
2006     }
2007
2008     private static class UnprocessedVpnInterfaceData {
2009         InstanceIdentifier<VpnInterface> identifier;
2010         VpnInterface vpnInterface;
2011
2012         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2013             super();
2014             this.identifier = identifier;
2015             this.vpnInterface = vpnInterface;
2016         }
2017
2018         @Override
2019         public int hashCode() {
2020             final int prime = 31;
2021             int result = 1;
2022             result = prime * result + ((identifier == null) ? 0 : identifier.hashCode());
2023             result = prime * result + ((vpnInterface == null) ? 0 : vpnInterface.hashCode());
2024             return result;
2025         }
2026
2027         @Override
2028         public boolean equals(Object obj) {
2029             if (this == obj) {
2030                 return true;
2031             }
2032             if (obj == null) {
2033                 return false;
2034             }
2035             if (getClass() != obj.getClass()) {
2036                 return false;
2037             }
2038             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2039             if (identifier == null) {
2040                 if (other.identifier != null) {
2041                     return false;
2042                 }
2043             } else if (!identifier.equals(other.identifier)) {
2044                 return false;
2045             }
2046             if (vpnInterface == null) {
2047                 if (other.vpnInterface != null) {
2048                     return false;
2049                 }
2050             } else if (!vpnInterface.equals(other.vpnInterface)) {
2051                 return false;
2052             }
2053             return true;
2054         }
2055     }
2056 }