vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Function;
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Preconditions;
  13 import com.google.common.base.Predicate;
  14 import com.google.common.base.Predicates;
  15 import com.google.common.collect.FluentIterable;
  16 import com.google.common.collect.Iterators;
  17 import com.google.common.util.concurrent.CheckedFuture;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.JdkFutureAdapters;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import java.math.BigInteger;
  23 import java.util.*;
  24 import java.util.concurrent.*;
  25
  26 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  27 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  28 import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
  29 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  30 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  31 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  32 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  33 import org.opendaylight.genius.mdsalutil.ActionInfo;
  34 import org.opendaylight.genius.mdsalutil.ActionType;
  35 import org.opendaylight.genius.mdsalutil.FlowEntity;
  36 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  37 import org.opendaylight.genius.mdsalutil.InstructionType;
  38 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  39 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  40 import org.opendaylight.genius.mdsalutil.MatchInfo;
  41 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  42 import org.opendaylight.genius.mdsalutil.NwConstants;
  43 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  44 import org.opendaylight.genius.utils.ServiceIndex;
  45 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  46 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  47 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  48 import org.opendaylight.netvirt.vpnmanager.intervpnlink.InterVpnLinkUtil;
  49 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  50 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  51 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  52 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  53 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.VpnTargets;
  54 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  55 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  56 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  57 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  58 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInputBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEvent;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEventBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEvent;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEventBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventData;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventDataBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventData;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventDataBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 125 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 126 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 127 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 128 import org.opendaylight.yangtools.yang.common.RpcError;
 129 import org.opendaylight.yangtools.yang.common.RpcResult;
 130 import org.slf4j.Logger;
 131 import org.slf4j.LoggerFactory;
 132
 133 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager>
 134         implements AutoCloseable {
 135     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 136     private final DataBroker dataBroker;
 137     private final IBgpManager bgpManager;
 138     private final IFibManager fibManager;
 139     private final IMdsalApiManager mdsalManager;
 140     private final IdManagerService idManager;
 141     private final OdlArputilService arpManager;
 142     private final OdlInterfaceRpcService ifaceMgrRpcService;
 143     private final NotificationPublishService notificationPublishService;
 144     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
 145     private ConcurrentHashMap<String, List<Runnable>> vpnInstanceToIdSynchronizerMap = new ConcurrentHashMap<String, List<Runnable>>();
 146     private ConcurrentHashMap<String, List<Runnable>> vpnInstanceOpDataSynchronizerMap = new ConcurrentHashMap<String, List<Runnable>>();
 147     private ExecutorService executorService = Executors.newSingleThreadExecutor();
 148     private static final int vpnInfUpdateTimerTaskDelay = 1000;
 149     private static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
 150     private BlockingQueue<UpdateData> vpnInterfacesUpdateQueue = new LinkedBlockingQueue<>();
 151     private ScheduledThreadPoolExecutor vpnInfUpdateTaskExecutor = (ScheduledThreadPoolExecutor) Executors
 152             .newScheduledThreadPool(1);
 153
 154     public VpnInterfaceManager(final DataBroker dataBroker,
 155                                final IBgpManager bgpManager,
 156                                final OdlArputilService arpManager,
 157                                final IdManagerService idManager,
 158                                final IMdsalApiManager mdsalManager,
 159                                final IFibManager fibManager,
 160                                final OdlInterfaceRpcService ifaceMgrRpcService,
 161                                final NotificationPublishService notificationPublishService) {
 162         super(VpnInterface.class, VpnInterfaceManager.class);
 163         this.dataBroker = dataBroker;
 164         this.bgpManager = bgpManager;
 165         this.arpManager = arpManager;
 166         this.idManager = idManager;
 167         this.mdsalManager = mdsalManager;
 168         this.fibManager = fibManager;
 169         this.ifaceMgrRpcService = ifaceMgrRpcService;
 170         this.notificationPublishService = notificationPublishService;
 171         vpnInfUpdateTaskExecutor.scheduleWithFixedDelay(new VpnInterfaceUpdateTimerTask(),
 172                 0, vpnInfUpdateTimerTaskDelay, TIME_UNIT);
 173     }
 174
 175     public Runnable isNotifyTaskQueued(String intfName) {
 176         return vpnIntfMap.remove(intfName);
 177     }
 178
 179     public void start() {
 180         LOG.info("{} start", getClass().getSimpleName());
 181         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 182     }
 183
 184     @Override
 185     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 186         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 187     }
 188
 189     @Override
 190     protected VpnInterfaceManager getDataTreeChangeListener() {
 191         return VpnInterfaceManager.this;
 192     }
 193
 194     public ConcurrentHashMap<String, List<Runnable>> getvpnInstanceToIdSynchronizerMap() {
 195         return vpnInstanceToIdSynchronizerMap;
 196     }
 197
 198     public ConcurrentHashMap<String, List<Runnable>> getvpnInstanceOpDataSynchronizerMap() {
 199         return vpnInstanceOpDataSynchronizerMap;
 200     }
 201
 202     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 203         return InstanceIdentifier.create(InterfacesState.class)
 204                 .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 205     }
 206
 207     @Override
 208     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 209         addVpnInterface(identifier, vpnInterface, null, null);
 210     }
 211
 212     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 213                                  final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 214         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 215         LOG.info("VPN Interface add event - intfName {}" ,vpnInterface.getName());
 216         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 217         final String interfaceName = key.getName();
 218
 219         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 220                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 221         if(interfaceState != null){
 222             try{
 223                 final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 224                 final int ifIndex = interfaceState.getIfIndex();
 225                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 226                 dataStoreCoordinator.enqueueJob("VPNINTERFACE-"+ interfaceName,
 227                         new Callable<List<ListenableFuture<Void>>>() {
 228                             @Override
 229                             public List<ListenableFuture<Void>> call() throws Exception {
 230                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 231                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 232                                 WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 233                                 processVpnInterfaceUp(dpnId, vpnInterface, ifIndex, false, writeConfigTxn, writeOperTxn, writeInvTxn);
 234                                 if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 235                                     LOG.trace("Adjacency changed upon VPNInterface {} Update for swapping VPN case",
 236                                             interfaceName);
 237                                     if (newAdjs != null) {
 238                                         for (Adjacency adj : newAdjs) {
 239                                             if (oldAdjs.contains(adj)) {
 240                                                 oldAdjs.remove(adj);
 241                                             } else {
 242                                                 addNewAdjToVpnInterface(identifier, adj, dpnId, writeOperTxn, writeConfigTxn);
 243                                             }
 244                                         }
 245                                     }
 246                                     for (Adjacency adj : oldAdjs) {
 247                                         delAdjFromVpnInterface(identifier, adj, dpnId, writeOperTxn, writeConfigTxn);
 248                                     }
 249                                 }
 250                                 List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
 251                                 futures.add(writeOperTxn.submit());
 252                                 futures.add(writeConfigTxn.submit());
 253                                 futures.add(writeInvTxn.submit());
 254                                 return futures;
 255                             }
 256                         });
 257             }catch (Exception e){
 258                 LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", interfaceName, e);
 259                 return;
 260             }
 261         } else if (vpnInterface.isIsRouterInterface()) {
 262             createVpnInterfaceForRouter(vpnInterface, interfaceName);
 263
 264         } else {
 265             LOG.error("Handling addition of VPN interface {} skipped as interfaceState is not available", interfaceName);
 266         }
 267     }
 268
 269     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface,
 270                                          final int lPortTag, boolean isInterfaceUp,
 271                                          WriteTransaction writeConfigTxn,
 272                                          WriteTransaction writeOperTxn,
 273                                          WriteTransaction writeInvTxn) {
 274
 275         final String interfaceName = vpnInterface.getName();
 276         if (!isInterfaceUp) {
 277             final String vpnName = vpnInterface.getVpnInstanceName();
 278             LOG.info("Binding vpn service to interface {} ", interfaceName);
 279             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 280             if (vpnId == VpnConstants.INVALID_ID) {
 281                 waitForVpnInstance(vpnName, VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS, vpnInstanceToIdSynchronizerMap);
 282                 vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 283                 if (vpnId == VpnConstants.INVALID_ID) {
 284                     LOG.error("VpnInstance to VPNId mapping not yet available for VpnName {} processing vpninterface {} " +
 285                             ", bailing out now.", vpnName, interfaceName);
 286                     return;
 287                 }
 288             } else {
 289                 // Incase of cluster reboot , VpnId would be available already as its a configDS fetch.
 290                 // However VpnInstanceOpData will be repopulated, so if its not available
 291                 // wait for 180 seconds and retry.
 292                 // TODO:  This wait to be removed by making vpnManager the central engine in carbon
 293                 String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 294                 VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
 295                 if (vpnInstanceOpDataEntry == null) {
 296                     LOG.debug("VpnInstanceOpData not yet populated for vpn {} rd {}", vpnName, vpnRd);
 297                     int retry = 2;
 298                     while (retry > 0) {
 299                         waitForVpnInstance(vpnName, VpnConstants.PER_VPN_INSTANCE_OPDATA_MAX_WAIT_TIME_IN_MILLISECONDS, vpnInstanceOpDataSynchronizerMap);
 300                         vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
 301                         if (vpnInstanceOpDataEntry != null) {
 302                             break;
 303                         }
 304                         retry--;
 305                         if (retry <= 0) {
 306                             LOG.error("VpnInstanceOpData not populated even after second retry for vpn {} rd {} vpninterface {}, bailing out ", vpnName, vpnRd, interfaceName);
 307                             return;
 308                         }
 309                     }
 310                 }
 311             }
 312
 313             boolean waitForVpnInterfaceOpRemoval = false;
 314             VpnInterface opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, vpnInterface.getName());
 315             if (opVpnInterface != null ) {
 316                 String opVpnName = opVpnInterface.getVpnInstanceName();
 317                 String primaryInterfaceIp = null;
 318                 if(opVpnName.equals(vpnName)) {
 319                     // Please check if the primary VRF Entry does not exist for VPNInterface
 320                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 321                     // back to back
 322                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 323                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 324                     if (adjs == null) {
 325                         LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained", interfaceName);
 326                         return;
 327                     }
 328                     for (Adjacency adj : adjs) {
 329                         if (adj.isPrimaryAdjacency()) {
 330                             primaryInterfaceIp = adj.getIpAddress();
 331                             break;
 332                         }
 333                     }
 334                     if (primaryInterfaceIp == null) {
 335                         LOG.info("VPN Interface {} addition failed as primary adjacency "
 336                                 + "for this vpn interface could not be obtained", interfaceName);
 337                         return;
 338                     }
 339                     // Get the rd of the vpn instance
 340                     String rd = getRouteDistinguisher(opVpnName);
 341                     rd =  (rd == null) ? opVpnName : rd;
 342                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, rd, primaryInterfaceIp);
 343                     if (vrf != null) {
 344                         LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
 345                         return;
 346                     }
 347                     waitForVpnInterfaceOpRemoval = true;
 348                 } else {
 349                     LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
 350                             interfaceName, vpnName, opVpnName);
 351                 }
 352             }
 353             if (!waitForVpnInterfaceOpRemoval) {
 354                 // Add the VPNInterface and quit
 355                 updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 356                 bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 357                 processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 358                 return;
 359             }
 360
 361             // FIB didn't get a chance yet to clean up this VPNInterface
 362             // Let us give it a chance here !
 363             LOG.info("Trying to add VPN Interface {}, but waiting for FIB to clean up! ", interfaceName);
 364             try {
 365                 Runnable notifyTask = new VpnNotifyTask();
 366                 vpnIntfMap.put(interfaceName, notifyTask);
 367                 synchronized (notifyTask) {
 368                     try {
 369                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 370                     } catch (InterruptedException e) {
 371                     }
 372                 }
 373             } finally {
 374                 vpnIntfMap.remove(interfaceName);
 375             }
 376
 377             opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
 378             if (opVpnInterface != null) {
 379                 LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
 380                 return;
 381             }
 382             // VPNInterface got removed, proceed with Add
 383             updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 384             bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 385             processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 386         } else {
 387             // Interface is retained in the DPN, but its Link Up.
 388             // Advertise prefixes again for this interface to BGP
 389             advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
 390                     vpnInterface);
 391         }
 392     }
 393
 394
 395 //    private class UpdateDpnToVpnWorker implements Callable<List<ListenableFuture<Void>>> {
 396 //        BigInteger dpnId;
 397 //        String vpnName;
 398 //        String interfaceName;
 399 //        boolean addToDpn;
 400 //        int lPortTag;
 401 //
 402 //        public UpdateDpnToVpnWorker(BigInteger dpnId, String vpnName, String interfaceName,
 403 //                                    int lPortTag, boolean addToDpn) {
 404 //            this.dpnId= dpnId;
 405 //            this.vpnName = vpnName;
 406 //            this.interfaceName = interfaceName;
 407 //            this.lPortTag = lPortTag;
 408 //            this.addToDpn = addToDpn;
 409 //        }
 410 //
 411 //        @Override
 412 //        public List<ListenableFuture<Void>> call() throws Exception {
 413 //            // If another renderer(for eg : CSS) needs to be supported, check can be performed here
 414 //            // to call the respective helpers.
 415 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 416 //            updateDpnDbs(dpnId, vpnName, interfaceName, addToDpn, writeTxn);
 417 //            List<ListenableFuture<Void>> futures = new ArrayList<>();
 418 //            futures.add(writeTxn.submit());
 419 //            ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
 420 //            Futures.addCallback(listenableFuture,
 421 //                    new UpdateDpnToVpnCallback(dpnId, vpnName, interfaceName, lPortTag, addToDpn));
 422 //            return futures;
 423 //        }
 424 //    }
 425 //
 426 //
 427 //    /**
 428 //     * JobCallback class is used as a future callback for
 429 //     * main and rollback workers to handle success and failure.
 430 //     */
 431 //    private class UpdateDpnToVpnCallback implements FutureCallback<List<Void>> {
 432 //        BigInteger dpnId;
 433 //        String vpnName;
 434 //        String interfaceName;
 435 //        boolean addToDpn;
 436 //        int lPortTag;
 437 //
 438 //        public UpdateDpnToVpnCallback(BigInteger dpnId, String vpnName, String interfaceName,
 439 //                                      int lPortTag, boolean addToDpn) {
 440 //            this.dpnId= dpnId;
 441 //            this.vpnName = vpnName;
 442 //            this.interfaceName = interfaceName;
 443 //            this.lPortTag = lPortTag;
 444 //            this.addToDpn = addToDpn;
 445 //        }
 446 //
 447 //        /**
 448 //         * @param voids
 449 //         * This implies that all the future instances have returned success. -- TODO: Confirm this
 450 //         */
 451 //        @Override
 452 //        public void onSuccess(List<Void> voids) {
 453 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 454 //            bindService(dpnId, vpnName, interfaceName, lPortTag, writeTxn);
 455 //            processVpnInterfaceAdjacencies(dpnId, vpnName, interfaceName, writeTxn);
 456 //            writeTxn.submit();
 457 //        }
 458 //
 459 //        /**
 460 //         *
 461 //         * @param throwable
 462 //         * This method is used to handle failure callbacks.
 463 //         * If more retry needed, the retrycount is decremented and mainworker is executed again.
 464 //         * After retries completed, rollbackworker is executed.
 465 //         * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 466 //         */
 467 //
 468 //        @Override
 469 //        public void onFailure(Throwable throwable) {
 470 //            LOG.warn("Job: failed with exception: ", throwable);
 471 //        }
 472 //    }
 473
 474
 475
 476
 477     private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
 478                                                  VpnInterface intf) {
 479         //Read NextHops
 480         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 481         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 482
 483         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 484         if (rd == null) {
 485             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 486                     intf.getName(), intf.getVpnInstanceName());
 487             return;
 488         } else {
 489             if (rd.equals(intf.getVpnInstanceName())) {
 490                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in " +
 491                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 492
 493                 return;
 494             }
 495         }
 496         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
 497                 intf.getVpnInstanceName(), rd);
 498
 499         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 500         if (nextHopIp == null){
 501             LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
 502             return;
 503         }
 504
 505         if (adjacencies.isPresent()) {
 506             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 507
 508             if (!nextHops.isEmpty()) {
 509                 LOG.trace("NextHops are " + nextHops);
 510                 for (Adjacency nextHop : nextHops) {
 511                     long label = nextHop.getLabel();
 512                     try {
 513                         LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 514                         bgpManager.advertisePrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
 515                         LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 516                     } catch(Exception e) {
 517                         LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
 518                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 519                     }
 520                 }
 521             }
 522         }
 523     }
 524
 525     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 526         //Read NextHops
 527         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 528         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 529
 530         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 531         if (rd == null) {
 532             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 533                     intf.getName(), intf.getVpnInstanceName());
 534             return;
 535         } else {
 536             if (rd.equals(intf.getVpnInstanceName())) {
 537                 LOG.info("withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in " +
 538                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 539                 return;
 540             }
 541         }
 542         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 543                 intf.getVpnInstanceName(), rd);
 544         if (adjacencies.isPresent()) {
 545             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 546
 547             if (!nextHops.isEmpty()) {
 548                 LOG.trace("NextHops are " + nextHops);
 549                 for (Adjacency nextHop : nextHops) {
 550                     try {
 551                         LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 552                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 553                         LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 554                     } catch(Exception e) {
 555                         LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
 556                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 557                     }
 558                 }
 559             }
 560         }
 561     }
 562
 563     public void updateVpnToDpnMapping(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 564         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 565         if (dpId == null) {
 566             dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, interfaceName);
 567         }
 568         if(!dpId.equals(BigInteger.ZERO)) {
 569             if(add) {
 570                 createOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 571             } else {
 572                 removeOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 573             }
 574         }
 575     }
 576
 577     private void bindService(BigInteger dpId, final String vpnInstanceName, final String vpnInterfaceName,
 578                              int lPortTag, WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
 579         final int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 580         final long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
 581
 582         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 583         dataStoreCoordinator.enqueueJob(vpnInterfaceName,
 584                 new Callable<List<ListenableFuture<Void>>>() {
 585                     @Override
 586                     public List<ListenableFuture<Void>> call() throws Exception {
 587                         WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 588                         int instructionKey = 0;
 589                         List<Instruction> instructions = new ArrayList<Instruction>();
 590
 591                         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(
 592                                 MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 593                         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_GW_MAC_TABLE, ++instructionKey));
 594
 595                         BoundServices
 596                                 serviceInfo =
 597                                 InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 598                                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX), priority,
 599                                         NwConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 600                         writeTxn.put(LogicalDatastoreType.CONFIGURATION,
 601                                 InterfaceUtils.buildServiceId(vpnInterfaceName, ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)), serviceInfo, true);
 602                         List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
 603                         futures.add(writeTxn.submit());
 604                         return futures;
 605                     }
 606                 });
 607         setupGwMacIfExternalVpn(dpId, vpnInterfaceName, vpnId, writeInvTxn, NwConstants.ADD_FLOW);
 608     }
 609
 610     private void setupGwMacIfExternalVpn(BigInteger dpnId, String interfaceName, long vpnId,
 611             WriteTransaction writeInvTxn, int addOrRemove) {
 612         InstanceIdentifier<VpnIds> vpnIdsInstanceIdentifier = VpnUtil.getVpnIdToVpnInstanceIdentifier(vpnId);
 613         Optional<VpnIds> vpnIdsOptional = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vpnIdsInstanceIdentifier);
 614         if (vpnIdsOptional.isPresent() && vpnIdsOptional.get().isExternalVpn()) {
 615             Optional<String> gwMacAddressOptional = InterfaceUtils.getMacAddressForInterface(dataBroker, interfaceName);
 616             if (!gwMacAddressOptional.isPresent()) {
 617                 LOG.error("Failed to get gwMacAddress for interface {}", interfaceName);
 618                 return;
 619             }
 620             String gwMacAddress = gwMacAddressOptional.get();
 621             FlowEntity flowEntity = VpnUtil.buildL3vpnGatewayFlow(dpnId, gwMacAddress, vpnId);
 622             if (addOrRemove == NwConstants.ADD_FLOW) {
 623                 mdsalManager.addFlowToTx(flowEntity, writeInvTxn);
 624             } else if (addOrRemove == NwConstants.DEL_FLOW) {
 625                 mdsalManager.removeFlowToTx(flowEntity, writeInvTxn);
 626             }
 627         }
 628     }
 629
 630     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, String vpnName, String interfaceName,
 631                                                 WriteTransaction writeConfigTxn,
 632                                                 WriteTransaction writeOperTxn) {
 633         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 634         // Read NextHops
 635         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 636         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 637
 638         if (adjacencies.isPresent()) {
 639             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 640             List<Adjacency> value = new ArrayList<>();
 641
 642             // Get the rd of the vpn instance
 643             String rd = getRouteDistinguisher(vpnName);
 644
 645             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 646             if (nextHopIp == null){
 647                 LOG.error("NextHop for interface {} is null", interfaceName);
 648                 return;
 649             }
 650
 651             List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
 652
 653             LOG.trace("NextHops for interface {} are {}", interfaceName, nextHops);
 654             for (Adjacency nextHop : nextHops) {
 655                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 656                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 657                         VpnUtil.getNextHopLabelKey((rd == null) ? vpnName
 658                                 : rd, prefix));
 659                 if (label == VpnConstants.INVALID_LABEL) {
 660                     LOG.error("Unable to fetch label from Id Manager. Bailing out of processing add/update of vpn interface {} for vpn {}", interfaceName, vpnName);
 661                     return;
 662                 }
 663                 List<String> adjNextHop = nextHop.getNextHopIpList();
 664                 value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIpList(
 665                         (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : Arrays.asList(nextHopIp))
 666                         .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 667
 668                 if (nextHop.isPrimaryAdjacency()) {
 669                     LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, interfaceName, vpnName);
 670                     writeOperTxn.merge(
 671                             LogicalDatastoreType.OPERATIONAL,
 672                             VpnUtil.getPrefixToInterfaceIdentifier(
 673                                     VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 674                             VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix), true);
 675                 } else {
 676                     //Extra route adjacency
 677                     LOG.trace("Adding prefix {} and nexthopList {} as extra-route for vpn", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName);
 678                     writeOperTxn.merge(
 679                             LogicalDatastoreType.OPERATIONAL,
 680                             VpnUtil.getVpnToExtrarouteIdentifier(
 681                                     (rd != null) ? rd : vpnName, nextHop.getIpAddress()),
 682                             VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()), true);
 683                 }
 684             }
 685
 686             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 687
 688             VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 689             InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 690             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, true);
 691             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 692
 693             for (Adjacency nextHop : aug.getAdjacency()) {
 694                 long label = nextHop.getLabel();
 695                 if (rd != null) {
 696                     addToLabelMapper(label, dpnId, nextHop.getIpAddress(), Arrays.asList(nextHopIp), vpnId,
 697                             interfaceName, null,false, rd, writeOperTxn);
 698                     addPrefixToBGP(rd, nextHop.getIpAddress(), nextHopIp, label, writeConfigTxn);
 699                     //TODO: ERT - check for VPNs importing my route
 700                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 701                         String vpnRd = vpn.getVrfId();
 702                         if (vpnRd != null) {
 703                             LOG.debug("Exporting route with rd {} prefix {} nexthop {} label {} to VPN {}", vpnRd, nextHop.getIpAddress(), nextHopIp, label, vpn);
 704                             fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), Arrays.asList(nextHopIp), (int) label,
 705                                     RouteOrigin.SELF_IMPORTED, writeConfigTxn);
 706                         }
 707                     }
 708                 } else {
 709                     // ### add FIB route directly
 710                     fibManager.addOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), Arrays.asList(nextHopIp),
 711                             (int) label, RouteOrigin.STATIC, writeConfigTxn);
 712                 }
 713             }
 714         }
 715     }
 716
 717     private List<VpnInstanceOpDataEntry> getVpnsImportingMyRoute(final String vpnName) {
 718         List<VpnInstanceOpDataEntry> vpnsToImportRoute = new ArrayList<>();
 719
 720         final String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 721         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
 722         if (vpnInstanceOpDataEntry == null) {
 723             LOG.debug("Could not retrieve vpn instance op data for {} to check for vpns importing the routes", vpnName);
 724             return vpnsToImportRoute;
 725         }
 726
 727         Predicate<VpnInstanceOpDataEntry> excludeVpn = new Predicate<VpnInstanceOpDataEntry>() {
 728             @Override
 729             public boolean apply(VpnInstanceOpDataEntry input) {
 730                 if (input.getVpnInstanceName() == null) {
 731                     LOG.error("Received vpn instance without identity");
 732                     return false;
 733                 }
 734                 return !input.getVpnInstanceName().equals(vpnName);
 735             }
 736         };
 737
 738         Predicate<VpnInstanceOpDataEntry> matchRTs = new Predicate<VpnInstanceOpDataEntry>() {
 739             @Override
 740             public boolean apply(VpnInstanceOpDataEntry input) {
 741                 Iterable<String> commonRTs = intersection(getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ExportExtcommunity),
 742                         getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 743                 return Iterators.size(commonRTs.iterator()) > 0;
 744             }
 745         };
 746
 747         Function<VpnInstanceOpDataEntry, String> toInstanceName = new Function<VpnInstanceOpDataEntry, String>() {
 748             @Override
 749             public String apply(VpnInstanceOpDataEntry vpnInstance) {
 750                 //return vpnInstance.getVpnInstanceName();
 751                 return vpnInstance.getVrfId();
 752             }
 753         };
 754
 755         vpnsToImportRoute = FluentIterable.from(VpnUtil.getAllVpnInstanceOpData(dataBroker)).
 756                 filter(excludeVpn).
 757                 filter(matchRTs).toList();
 758         return vpnsToImportRoute;
 759     }
 760
 761     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
 762         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
 763
 764         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 765         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
 766         if (vpnInstanceOpDataEntry == null) {
 767             LOG.debug("Could not retrieve vpn instance op data for {} to check for vpns exporting the routes", vpnName);
 768             return vpnsToExportRoute;
 769         }
 770
 771         Predicate<VpnInstanceOpDataEntry> excludeVpn = new Predicate<VpnInstanceOpDataEntry>() {
 772             @Override
 773             public boolean apply(VpnInstanceOpDataEntry input) {
 774                 if (input.getVpnInstanceName() == null) {
 775                     LOG.error("Received vpn instance without identity");
 776                     return false;
 777                 }
 778                 return !input.getVpnInstanceName().equals(vpnName);
 779             }
 780         };
 781
 782         Predicate<VpnInstanceOpDataEntry> matchRTs = new Predicate<VpnInstanceOpDataEntry>() {
 783             @Override
 784             public boolean apply(VpnInstanceOpDataEntry input) {
 785                 Iterable<String> commonRTs = intersection(getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
 786                         getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
 787                 return Iterators.size(commonRTs.iterator()) > 0;
 788             }
 789         };
 790
 791         Function<VpnInstanceOpDataEntry, String> toInstanceName = new Function<VpnInstanceOpDataEntry, String>() {
 792             @Override
 793             public String apply(VpnInstanceOpDataEntry vpnInstance) {
 794                 return vpnInstance.getVpnInstanceName();
 795             }
 796         };
 797
 798         vpnsToExportRoute = FluentIterable.from(VpnUtil.getAllVpnInstanceOpData(dataBroker)).
 799                 filter(excludeVpn).
 800                 filter(matchRTs).toList();
 801         return vpnsToExportRoute;
 802     }
 803
 804     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
 805         final Predicate<T> inPredicate = Predicates.<T>in(collection2);
 806         return new Iterable<T>() {
 807             @Override
 808             public Iterator<T> iterator() {
 809                 return Iterators.filter(collection1.iterator(), inPredicate);
 810             }
 811         };
 812     }
 813
 814     private List<String> getRts(VpnInstanceOpDataEntry vpnInstance, VpnTarget.VrfRTType rtType) {
 815         String name = vpnInstance.getVpnInstanceName();
 816         List<String> rts = new ArrayList<>();
 817         org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnTargets targets = vpnInstance.getVpnTargets();
 818         if (targets == null) {
 819             LOG.trace("vpn targets not available for {}", name);
 820             return rts;
 821         }
 822         List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget> vpnTargets = targets.getVpnTarget();
 823         if (vpnTargets == null) {
 824             LOG.trace("vpnTarget values not available for {}", name);
 825             return rts;
 826         }
 827         for (org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget target : vpnTargets) {
 828             //TODO: Check for RT type is Both
 829             if(target.getVrfRTType().equals(rtType) ||
 830                     target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
 831                 String rtValue = target.getVrfRTValue();
 832                 rts.add(rtValue);
 833             }
 834         }
 835         return rts;
 836     }
 837
 838     private List<String> getExportRts(VpnInstance vpnInstance) {
 839         List<String> exportRts = new ArrayList<>();
 840         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 841         VpnTargets targets = vpnConfig.getVpnTargets();
 842         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 843         for (VpnTarget target : vpnTargets) {
 844             if (target.getVrfRTType().equals(VpnTarget.VrfRTType.ExportExtcommunity)) {
 845                 String rtValue = target.getVrfRTValue();
 846                 exportRts.add(rtValue);
 847             }
 848         }
 849         return exportRts;
 850     }
 851
 852     private String getRouteDistinguisher(String vpnName) {
 853         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 854                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 855         Optional<VpnInstance> vpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 856         String rd = null;
 857         if(vpnInstance.isPresent()) {
 858             VpnInstance instance = vpnInstance.get();
 859             org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig config = instance.getIpv4Family();
 860             rd = config.getRouteDistinguisher();
 861         }
 862         return rd;
 863     }
 864
 865     /**
 866      * JobCallback class is used as a future callback for
 867      * main and rollback workers to handle success and failure.
 868      */
 869     private class DpnEnterExitVpnWorker implements FutureCallback<List<Void>> {
 870         BigInteger dpnId;
 871         String vpnName;
 872         String rd;
 873         boolean entered;
 874
 875         public DpnEnterExitVpnWorker(BigInteger dpnId, String vpnName, String rd, boolean entered) {
 876             this.entered = entered;
 877             this.dpnId = dpnId;
 878             this.vpnName = vpnName;
 879             this.rd = rd;
 880         }
 881
 882         /**
 883          * @param voids
 884          * This implies that all the future instances have returned success. -- TODO: Confirm this
 885          */
 886         @Override
 887         public void onSuccess(List<Void> voids) {
 888             if (entered) {
 889                 publishAddNotification(dpnId, vpnName, rd);
 890             } else {
 891                 publishRemoveNotification(dpnId, vpnName, rd);
 892             }
 893         }
 894
 895         /**
 896          *
 897          * @param throwable
 898          * This method is used to handle failure callbacks.
 899          * If more retry needed, the retrycount is decremented and mainworker is executed again.
 900          * After retries completed, rollbackworker is executed.
 901          * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 902          */
 903         @Override
 904         public void onFailure(Throwable throwable) {
 905             LOG.warn("Job: failed with exception: ", throwable);
 906         }
 907     }
 908
 909     private void createOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 910         String routeDistinguisher = getRouteDistinguisher(vpnName);
 911         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 912         Boolean newDpnOnVpn = Boolean.FALSE;
 913
 914         synchronized (vpnName.intern()) {
 915             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 916             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 917             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 918             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data
 919                     .entry.vpn.to.dpn.list.VpnInterfaces
 920                     vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 921
 922             if (dpnInVpn.isPresent()) {
 923                 VpnToDpnList vpnToDpnList = dpnInVpn.get();
 924                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 925                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = vpnToDpnList.getVpnInterfaces();
 926                 if (vpnInterfaces == null) {
 927                     vpnInterfaces = new ArrayList<>();
 928                 }
 929                 vpnInterfaces.add(vpnInterface);
 930                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder(vpnToDpnList);
 931                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 932
 933                 if (writeTxn != null) {
 934                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 935                 } else {
 936                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 937                 }
 938                 /* If earlier state was inactive, it is considered new DPN coming back to the
 939                  * same VPN
 940                  */
 941                 if (vpnToDpnList.getDpnState() == VpnToDpnList.DpnState.Inactive) {
 942                     newDpnOnVpn = Boolean.TRUE;
 943                 }
 944             } else {
 945                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 946                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = new ArrayList<>();
 947                 vpnInterfaces.add(vpnInterface);
 948                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder().setDpnId(dpnId);
 949                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 950
 951                 if (writeTxn != null) {
 952                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 953                 } else {
 954                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 955                 }
 956                 newDpnOnVpn = Boolean.TRUE;
 957             }
 958             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
 959             try {
 960                 futures.get();
 961             } catch (InterruptedException | ExecutionException e) {
 962                 LOG.error("Error adding to dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
 963                 throw new RuntimeException(e.getMessage());
 964             }
 965         }
 966         /*
 967          * Informing the Fib only after writeTxn is submitted successfuly.
 968          */
 969         if (newDpnOnVpn) {
 970             LOG.debug("Sending populateFib event for new dpn {} in VPN {}", dpnId, vpnName);
 971             fibManager.populateFibOnNewDpn(dpnId, vpnId, rd, new DpnEnterExitVpnWorker(dpnId, vpnName, rd, true /* entered */));
 972         }
 973     }
 974
 975     private void removeOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 976         Boolean lastDpnOnVpn = Boolean.FALSE;
 977         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
 978         synchronized (vpnName.intern()) {
 979             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 980             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 981             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 982             if (dpnInVpn.isPresent()) {
 983                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 984                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 985                 org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 986                         currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 987
 988                 if (vpnInterfaces.remove(currVpnInterface)) {
 989                     if (vpnInterfaces.isEmpty()) {
 990                         List<IpAddresses> ipAddresses = dpnInVpn.get().getIpAddresses();
 991                         if (ipAddresses == null || ipAddresses.isEmpty()) {
 992                             VpnToDpnListBuilder dpnInVpnBuilder =
 993                                     new VpnToDpnListBuilder(dpnInVpn.get())
 994                                             .setDpnState(VpnToDpnList.DpnState.Inactive)
 995                                             .setVpnInterfaces(null);
 996                             if (writeTxn != null) {
 997                                 writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build(), true);
 998                             } else {
 999                                 VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build());
1000                             }
1001                             lastDpnOnVpn = Boolean.TRUE;
1002                         } else {
1003                             LOG.warn("vpn interfaces are empty but ip addresses are present for the vpn {} in dpn {}", vpnName, dpnId);
1004                         }
1005                     } else {
1006                         if (writeTxn != null) {
1007                             writeTxn.delete(LogicalDatastoreType.OPERATIONAL, id.child(
1008                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1009                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1010                                     new VpnInterfacesKey(intfName)));
1011                         } else {
1012                             VpnUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1013                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1014                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1015                                     new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
1016                         }
1017                     }
1018                 }
1019             }
1020             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
1021             try {
1022                 futures.get();
1023             } catch (InterruptedException | ExecutionException e) {
1024                 LOG.error("Error removing from dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
1025                 throw new RuntimeException(e.getMessage());
1026             }
1027         }
1028         if (lastDpnOnVpn) {
1029             LOG.debug("Sending cleanup event for dpn {} in VPN {}", dpnId, vpnName);
1030             fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd, new DpnEnterExitVpnWorker(dpnId, vpnName, rd, false /* exited */));
1031         }
1032     }
1033
1034     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1035         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1036         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1037             String rd = vpn.getVrfId();
1038             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1039             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1040             if (vrfEntries != null) {
1041                 for (VrfEntry vrfEntry : vrfEntries) {
1042                     try {
1043                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1044                             continue;
1045                         }
1046                         String prefix = vrfEntry.getDestPrefix();
1047                         long label = vrfEntry.getLabel();
1048                         List<String> nextHops = vrfEntry.getNextHopAddressList();
1049                         SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1050                         for (String nh : nextHops) {
1051                             if (route != null) {
1052                                 LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1053                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, (int)label, route, writeConfigTxn);
1054                             } else {
1055                                 LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1056                                 fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, prefix, Arrays.asList(nh), (int)label,
1057                                         RouteOrigin.SELF_IMPORTED, writeConfigTxn);
1058                             }
1059                         }
1060                     } catch (Exception e) {
1061                         LOG.error("Exception occurred while importing route with prefix {} label {} nexthop {} from vpn {} to vpn {}", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpn.getVpnInstanceName(), vpnName);
1062                     }
1063                 }
1064                 writeConfigTxn.submit();
1065             } else {
1066                 LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getVrfId());
1067             }
1068         }
1069     }
1070
1071     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label, WriteTransaction writeConfigTxn) {
1072         try {
1073             LOG.info("ADD: Adding Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1074             fibManager.addOrUpdateFibEntry(dataBroker, rd, prefix, Arrays.asList(nextHopIp), (int)label, RouteOrigin.STATIC, writeConfigTxn);
1075             bgpManager.advertisePrefix(rd, prefix, Arrays.asList(nextHopIp), (int)label);
1076             LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1077         } catch(Exception e) {
1078             LOG.error("Add prefix failed", e);
1079         }
1080     }
1081
1082     @Override
1083     public void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1084         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
1085         LOG.info("VPN Interface remove event - intfName {}" ,vpnInterface.getName());
1086         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1087         final String interfaceName = key.getName();
1088
1089         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1090         final Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1091         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
1092                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1093         if (optVpnInterface.isPresent()){
1094             BigInteger dpnId = BigInteger.ZERO;
1095             Boolean dpnIdRetrieved = Boolean.FALSE;
1096             if(interfaceState != null){
1097                 try{
1098                     dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1099                     dpnIdRetrieved = Boolean.TRUE;
1100                 }catch (Exception e){
1101                     LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", interfaceName, e);
1102                 }
1103             } else {
1104                 LOG.error("Unable to retrieve interfaceState for interface {} , quitting ", interfaceName);
1105                 return;
1106             }
1107             final VpnInterface vpnOpInterface = optVpnInterface.get();
1108             if(dpnIdRetrieved == Boolean.FALSE){
1109                 LOG.info("dpnId for {} has not been retrieved yet. Fetching from vpn interface operational DS", interfaceName);
1110                 dpnId = vpnOpInterface.getDpnId();
1111             }
1112             final int ifIndex = interfaceState.getIfIndex();
1113             final BigInteger dpId = dpnId;
1114             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1115             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1116                     new Callable<List<ListenableFuture<Void>>>() {
1117                         @Override
1118                         public List<ListenableFuture<Void>> call() throws Exception {
1119                             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1120                             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1121                             WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
1122                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, false, true, writeConfigTxn, writeOperTxn, writeInvTxn);
1123                             List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
1124                             futures.add(writeOperTxn.submit());
1125                             futures.add(writeConfigTxn.submit());
1126                             futures.add(writeInvTxn.submit());
1127                             return futures;
1128                         }
1129                     });
1130
1131         } else if (vpnInterface.isIsRouterInterface()) {
1132
1133             List<Adjacency> adjsList = new ArrayList<>();
1134             Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
1135             if (adjs != null) {
1136                 adjsList = adjs.getAdjacency();
1137                 for (Adjacency adj : adjsList) {
1138                     if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
1139                         String primaryInterfaceIp = adj.getIpAddress();
1140                         String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1141                         fibManager.removeFibEntry(dataBroker, vpnInterface.getVpnInstanceName(), prefix, null);
1142                         return;
1143                     }
1144                 }
1145             }
1146         } else {
1147             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event",
1148                     interfaceName);
1149         }
1150     }
1151
1152     protected void processVpnInterfaceDown(BigInteger dpId,
1153                                            String interfaceName,
1154                                            int lPortTag,
1155                                            boolean isInterfaceStateDown,
1156                                            boolean isConfigRemoval,
1157                                            WriteTransaction writeConfigTxn,
1158                                            WriteTransaction writeOperTxn,
1159                                            WriteTransaction writeInvTxn) {
1160         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1161         if (!isInterfaceStateDown) {
1162             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1163             if(vpnInterface == null){
1164                 LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
1165                 return;
1166             }else{
1167                 final String vpnName = vpnInterface.getVpnInstanceName();
1168                 if(!vpnInterface.isScheduledForRemove()){
1169                     VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE, writeOperTxn);
1170                     removeAdjacenciesFromVpn(dpId, interfaceName, vpnInterface.getVpnInstanceName(), writeConfigTxn);
1171                     LOG.info("Unbinding vpn service from interface {} ", interfaceName);
1172                     unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown, isConfigRemoval, writeConfigTxn, writeInvTxn);
1173                 }else{
1174                     LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ", interfaceName);
1175                     return;
1176                 }
1177             }
1178         } else {
1179             // Interface is retained in the DPN, but its Link Down.
1180             // Only withdraw the prefixes for this interface from BGP
1181             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1182             if(vpnInterface == null){
1183                 LOG.info("Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in operational data store", interfaceName);
1184                 return;
1185             }else {
1186                 withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
1187             }
1188         }
1189     }
1190
1191     private void waitForFibToRemoveVpnPrefix(String interfaceName) {
1192         // FIB didn't get a chance yet to clean up this VPNInterface
1193         // Let us give it a chance here !
1194         LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
1195         try {
1196             Runnable notifyTask = new VpnNotifyTask();
1197             vpnIntfMap.put(interfaceName, notifyTask);
1198             synchronized (notifyTask) {
1199                 try {
1200                     notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
1201                 } catch (InterruptedException e) {
1202                 }
1203             }
1204         } finally {
1205             vpnIntfMap.remove(interfaceName);
1206         }
1207     }
1208
1209     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final String interfaceName, final String vpnName,
1210                                           WriteTransaction writeConfigTxn) {
1211         //Read NextHops
1212         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1213         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1214         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1215
1216         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1217         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", interfaceName,
1218                 vpnName, rd);
1219         if (adjacencies.isPresent()) {
1220             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1221
1222             if (!nextHops.isEmpty()) {
1223                 LOG.trace("NextHops are " + nextHops);
1224                 for (Adjacency nextHop : nextHops) {
1225                     List<String> nhList = new ArrayList<String>();
1226                     if (nextHop.isPrimaryAdjacency()) {
1227                         // This is either an extra-route (or) a learned IP via subnet-route
1228                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1229                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1230                             LOG.error("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
1231                                     rd, nextHop.getIpAddress());
1232                             continue;
1233                         }
1234                         nhList = Arrays.asList(nextHopIp);
1235                     } else {
1236                         // This is a primary adjacency
1237                         nhList = nextHop.getNextHopIpList();
1238                     }
1239                     if (rd.equals(vpnName)) {
1240                         //this is an internal vpn - the rd is assigned to the vpn instance name;
1241                         //remove from FIB directly
1242                         for(String nh : nhList) {
1243                             fibManager.removeOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), nh, writeConfigTxn);
1244                         }
1245                     } else {
1246                         List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1247                         for (String nh : nhList) {
1248                             //IRT: remove routes from other vpns importing it
1249                             removePrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1250                             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1251                                 String vpnRd = vpn.getVrfId();
1252                                 if (vpnRd != null) {
1253                                     LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd, nextHop.getIpAddress(), vpn.getVpnInstanceName());
1254                                     fibManager.removeOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), nh, writeConfigTxn);
1255                                 }
1256                             }
1257                         }
1258                     }
1259                     String ip = nextHop.getIpAddress().split("/")[0];
1260                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
1261                             vpnName, ip);
1262                     if (vpnPortipToPort != null && !vpnPortipToPort.isConfig()) {
1263                         LOG.trace("VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
1264                                 vpnPortipToPort.getPortName(),ip);
1265                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip);
1266                     }
1267                 }
1268             }
1269         }
1270     }
1271
1272
1273     private void unbindService(BigInteger dpId, String vpnInstanceName, final String vpnInterfaceName,
1274                                int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval,
1275                                WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
1276         short l3vpn_service_index = ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX);
1277         if (!isInterfaceStateDown && isConfigRemoval) {
1278             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1279             dataStoreCoordinator.enqueueJob(vpnInterfaceName,
1280                     new Callable<List<ListenableFuture<Void>>>() {
1281                         @Override
1282                         public List<ListenableFuture<Void>> call() throws Exception {
1283                             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
1284                             writeTxn.delete(LogicalDatastoreType.CONFIGURATION,
1285                                     InterfaceUtils.buildServiceId(vpnInterfaceName,
1286                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)));
1287
1288                             List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
1289                             futures.add(writeTxn.submit());
1290                             return futures;
1291                         }
1292                     });
1293         }
1294         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
1295         setupGwMacIfExternalVpn(dpId, vpnInterfaceName, vpnId, writeConfigTxn, NwConstants.DEL_FLOW);
1296     }
1297
1298
1299     private void removePrefixFromBGP(String rd, String prefix, String nextHop, WriteTransaction writeConfigTxn) {
1300         try {
1301             LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
1302             fibManager.removeOrUpdateFibEntry(dataBroker, rd, prefix, nextHop, writeConfigTxn);
1303             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1304             LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
1305         } catch(Exception e) {
1306             LOG.error("Delete prefix failed", e);
1307         }
1308     }
1309
1310     @Override
1311     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original, final VpnInterface update) {
1312         LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
1313         LOG.info("VPN Interface update event - intfName {}" ,update.getName());
1314         final String oldVpnName = original.getVpnInstanceName();
1315         final String newVpnName = update.getVpnInstanceName();
1316         final BigInteger dpnId = update.getDpnId();
1317         final UpdateData updateData = new UpdateData(identifier, original, update);
1318         final List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency() != null ? original
1319                 .getAugmentation(Adjacencies.class).getAdjacency() : new ArrayList<Adjacency>();
1320         final List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency() != null ? update
1321                 .getAugmentation(Adjacencies.class).getAdjacency() : new ArrayList<Adjacency>();
1322
1323         //handles switching between <internal VPN - external VPN>
1324         if (!oldVpnName.equals(newVpnName)) {
1325             vpnInterfacesUpdateQueue.add(updateData);
1326             LOG.trace("UpdateData on VPNInterface {} update upon VPN swap added to update queue",
1327                     updateData.getOriginal().getName());
1328             return;
1329         }
1330         final DataStoreJobCoordinator vpnInfAdjUpdateDataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1331         vpnInfAdjUpdateDataStoreCoordinator.enqueueJob("VPNINTERFACE-" + update.getName(), new Callable<List<ListenableFuture<Void>>>() {
1332             @Override
1333             public List<ListenableFuture<Void>> call() throws Exception {
1334                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1335                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1336                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1337                 //handle both addition and removal of adjacencies
1338                 //currently, new adjacency may be an extra route
1339                 if (!oldAdjs.equals(newAdjs)) {
1340                     for (Adjacency adj : newAdjs) {
1341                         if (oldAdjs.contains(adj)) {
1342                             oldAdjs.remove(adj);
1343                         } else {
1344                             // add new adjacency - right now only extra route will hit this path
1345                             addNewAdjToVpnInterface(identifier, adj, dpnId, writeOperTxn, writeConfigTxn);
1346                         }
1347                     }
1348                     for (Adjacency adj : oldAdjs) {
1349                         delAdjFromVpnInterface(identifier, adj, dpnId, writeOperTxn, writeConfigTxn);
1350                     }
1351                 }
1352                 futures.add(writeOperTxn.submit());
1353                 futures.add(writeConfigTxn.submit());
1354                 return futures;
1355             }
1356         });
1357     }
1358
1359     class VpnInterfaceUpdateTimerTask extends TimerTask {
1360
1361         @Override
1362         public void run() {
1363             List<UpdateData> processQueue = new ArrayList<>();
1364             vpnInterfacesUpdateQueue.drainTo(processQueue);
1365             for (UpdateData updData : processQueue) {
1366                 remove(updData.getIdentifier(), updData.getOriginal());
1367                 //TODO: Refactor wait to be based on queue size
1368                 waitForFibToRemoveVpnPrefix(updData.getUpdate().getName());
1369                 LOG.trace("Processed Remove for update on VPNInterface {} upon VPN swap",
1370                         updData.getOriginal().getName());
1371             }
1372             for (UpdateData updData : processQueue) {
1373                 final List<Adjacency> oldAdjs = updData.getOriginal().getAugmentation(Adjacencies.class).
1374                         getAdjacency() != null ? updData.getOriginal().getAugmentation(Adjacencies.class).getAdjacency()
1375                         : new ArrayList<Adjacency>();
1376                 final List<Adjacency> newAdjs = updData.getUpdate().getAugmentation(Adjacencies.class).
1377                         getAdjacency() != null ? updData.getUpdate().getAugmentation(Adjacencies.class).getAdjacency()
1378                         : new ArrayList<Adjacency>();
1379                 addVpnInterface(updData.getIdentifier(), updData.getUpdate(), oldAdjs, newAdjs);
1380                 LOG.trace("Processed Add for update on VPNInterface {} upon VPN swap",
1381                         updData.getUpdate().getName());
1382             }
1383         }
1384     }
1385
1386
1387     public void processArpRequest(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715
1388                                           .IpAddress srcIP, PhysAddress srcMac, org.opendaylight.yang.gen.v1.urn.ietf
1389             .params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress targetIP, PhysAddress targetMac, String srcInterface){
1390         //Build ARP response with ARP requests TargetIp TargetMac as the Arp Response SrcIp and SrcMac
1391         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
1392                 .setDstIpaddress(srcIP).setDstMacaddress(srcMac).setSrcIpaddress(targetIP).setSrcMacaddress(targetMac).build();
1393         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
1394         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
1395         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
1396             @Override
1397             public void onFailure(Throwable error) {
1398                 LOG.error("Error - {}", msgFormat, error);
1399             }
1400
1401             @Override
1402             public void onSuccess(RpcResult<Void> result) {
1403                 if(!result.isSuccessful()) {
1404                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
1405                 } else {
1406                     LOG.debug("Successful RPC Result - {}", msgFormat);
1407                 }
1408             }
1409         });
1410     }
1411
1412     private String getErrorText(Collection<RpcError> errors) {
1413         StringBuilder errorText = new StringBuilder();
1414         for(RpcError error : errors) {
1415             errorText.append(",").append(error.getErrorType()).append("-")
1416                     .append(error.getMessage());
1417         }
1418         return errorText.toString();
1419     }
1420
1421     private void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1422                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd,
1423                                   WriteTransaction writeOperTxn) {
1424         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1425         Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
1426         Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
1427         Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
1428         if (!isSubnetRoute) {
1429             // NextHop must be present for non-subnetroute entries
1430             Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
1431         }
1432         LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
1433         if (dpnId != null) {
1434             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1435                     .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
1436             LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1437             lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(vpnId)
1438                     .setIsSubnetRoute(isSubnetRoute);
1439             if (elanTag != null) {
1440                 lriBuilder.setElanTag(elanTag);
1441             }
1442             if (vpnInterfaceName != null) {
1443                 lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1444             }
1445             lriBuilder.setParentVpnRd(rd);
1446             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
1447             if (vpnInstanceOpDataEntry != null) {
1448                 List<String> vpnInstanceNames = Arrays.asList(vpnInstanceOpDataEntry.getVpnInstanceName());
1449                 lriBuilder.setVpnInstanceList(vpnInstanceNames);
1450             }
1451             LabelRouteInfo lri = lriBuilder.build();
1452             LOG.trace("Adding route info to label map: {}", lri);
1453             if (writeOperTxn != null) {
1454                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, lriIid, lri, true);
1455             } else {
1456                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
1457             }
1458         } else {
1459             LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
1460         }
1461     }
1462
1463     public void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
1464                                                         long elantag, BigInteger dpnId, WriteTransaction writeTxn) {
1465         SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
1466         RouteOrigin origin = RouteOrigin.STATIC; // Only case when a route is considered as directly connected
1467         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1468                 .setLabel((long)label).setOrigin(origin.getValue())
1469                 .addAugmentation(SubnetRoute.class, route).build();
1470
1471         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
1472
1473         //TODO: What should be parentVpnId? Get it from RD?
1474         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1475         addToLabelMapper((long)label, dpnId, prefix, Arrays.asList(nextHop), vpnId, null, elantag, true, rd, null);
1476         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1477
1478         InstanceIdentifierBuilder<VrfTables> idBuilder =
1479                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1480         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1481
1482         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1483                 setVrfEntry(vrfEntryList).build();
1484
1485         if (writeTxn != null) {
1486             writeTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1487         } else {
1488             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1489         }
1490
1491         List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1492         if (vpnsToImportRoute.size() > 0) {
1493             VrfEntry importingVrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1494                     .setLabel((long) label).setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
1495                     .addAugmentation(SubnetRoute.class, route).build();
1496             List<VrfEntry> importingVrfEntryList = Arrays.asList(importingVrfEntry);
1497             for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
1498                 LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop, label, vpnInstance.getVpnInstanceName());
1499                 String importingRd = vpnInstance.getVrfId();
1500                 InstanceIdentifier<VrfTables> importingVrfTableId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(importingRd)).build();
1501                 VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(importingVrfEntryList).build();
1502                 if (writeTxn != null) {
1503                     writeTxn.merge(LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable, true);
1504                 } else {
1505                     VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable);
1506                 }
1507             }
1508         }
1509     }
1510
1511     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1512                                                         SubnetRoute route, WriteTransaction writeConfigTxn) {
1513
1514         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1515         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1516                 .setLabel((long)label).setOrigin(origin.getValue())
1517                 .addAugmentation(SubnetRoute.class, route).build();
1518         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, route.getElantag());
1519         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1520         InstanceIdentifierBuilder<VrfTables> idBuilder =
1521                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1522         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1523         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1524                 setVrfEntry(vrfEntryList).build();
1525         if (writeConfigTxn != null) {
1526             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1527         } else {
1528             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1529         }
1530     }
1531
1532     public void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName){
1533         fibManager.removeFibEntry(dataBroker, rd, prefix, null);
1534         List<VpnInstanceOpDataEntry> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1535         for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
1536             String importingRd = vpnInstance.getVrfId();
1537             LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix, vpnInstance.getVpnInstanceName());
1538             fibManager.removeFibEntry(dataBroker, importingRd, prefix, null);
1539         }
1540     }
1541
1542     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1543
1544         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1545
1546         if (optVpnInterface.isPresent()) {
1547             VpnInterface currVpnIntf = optVpnInterface.get();
1548             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1549             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1550
1551             rd = (rd != null) ? rd : currVpnIntf.getVpnInstanceName();
1552             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1553             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, adjPath);
1554             long label =
1555                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1556                             VpnUtil.getNextHopLabelKey(rd, prefix));
1557             if (label == 0) {
1558                 LOG.error("Unable to fetch label from Id Manager. Bailing out of adding new adjacency {} to vpn interface {} for vpn {}", adj.getIpAddress(), currVpnIntf.getName(), currVpnIntf.getVpnInstanceName());
1559                 return;
1560             }
1561             List<Adjacency> adjacencies;
1562             if (optAdjacencies.isPresent()) {
1563                 adjacencies = optAdjacencies.get().getAdjacency();
1564             } else {
1565                 //This code will not be hit since VM adjacency will always be there
1566                 adjacencies = new ArrayList<>();
1567             }
1568
1569             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIpList(adj.getNextHopIpList())
1570                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
1571
1572             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1573             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove());
1574
1575             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1576             if (adj.getNextHopIpList() != null) {
1577                 for (String nh : adj.getNextHopIpList()) {
1578                     addExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(), (int) label,
1579                             currVpnIntf.getName(), writeOperTxn, writeConfigTxn);
1580                 }
1581             }
1582         }
1583     }
1584
1585     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1586         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1587
1588         if (optVpnInterface.isPresent()) {
1589             VpnInterface currVpnIntf = optVpnInterface.get();
1590
1591             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1592             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1593             if (optAdjacencies.isPresent()) {
1594                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1595
1596                 if (!adjacencies.isEmpty()) {
1597                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1598                     rd = (rd != null) ? rd :currVpnIntf.getVpnInstanceName();
1599                     LOG.trace("Adjacencies are " + adjacencies);
1600                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1601                     while (adjIt.hasNext()) {
1602                         Adjacency adjElem = adjIt.next();
1603                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1604                             adjIt.remove();
1605
1606                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1607                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1608                                     currVpnIntf.getVpnInstanceName(),
1609                                     aug, dpnId, currVpnIntf.isScheduledForRemove());
1610
1611                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1612                             if (adj.getNextHopIpList() != null) {
1613                                 for (String nh : adj.getNextHopIpList()) {
1614                                     delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
1615                                             currVpnIntf.getName(), writeConfigTxn);
1616                                 }
1617                             }
1618                             break;
1619                         }
1620
1621                     }
1622                 }
1623             }
1624         }
1625
1626     }
1627
1628     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label,
1629                                  String intfName, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1630
1631         Boolean writeOperTxnPresent = true;
1632         Boolean writeConfigTxnPresent = true;
1633         if (writeOperTxn == null) {
1634             writeOperTxnPresent = false;
1635             writeOperTxn = dataBroker.newWriteOnlyTransaction();
1636         }
1637         if (writeConfigTxn == null) {
1638             writeConfigTxnPresent = false;
1639             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1640         }
1641
1642         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1643         writeOperTxn.merge(
1644                 LogicalDatastoreType.OPERATIONAL,
1645                 VpnUtil.getVpnToExtrarouteIdentifier( (rd != null) ? rd : routerID, destination),
1646                 VpnUtil.getVpnToExtraroute(destination, Arrays.asList(nextHop)), true);
1647
1648         BigInteger dpnId = null;
1649         if (intfName != null && !intfName.isEmpty()) {
1650             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1651             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1652             if (nextHopIp == null || nextHopIp.isEmpty()) {
1653                 LOG.error("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1654                         intfName, destination);
1655                 return;
1656             }
1657             nextHop = nextHopIp;
1658         }
1659         List<String> nextHopIpList = Arrays.asList(nextHop);
1660         if (rd != null) {
1661             /* Label mapper is required only for BGP VPN and not for Internal VPN */
1662             addToLabelMapper((long) label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(dataBroker, routerID),
1663                     intfName, null, false, rd, null);
1664         }
1665
1666         // TODO (eperefr): This is a limitation to be stated in docs. When configuring static route to go to
1667         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1668         // first place.
1669         Optional<InterVpnLink> optInterVpnLink = InterVpnLinkUtil.getInterVpnLinkByEndpointIp(dataBroker, nextHop);
1670         if ( optInterVpnLink.isPresent() ) {
1671             InterVpnLink interVpnLink = optInterVpnLink.get();
1672             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1673             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1674             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1675             // This is like leaking one of the Vpn2 routes towards Vpn1
1676             boolean nexthopIsVpn2 = ( interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nextHop) );
1677             String srcVpnUuid = (nexthopIsVpn2) ? interVpnLink.getSecondEndpoint().getVpnUuid().getValue()
1678                     : interVpnLink.getFirstEndpoint().getVpnUuid().getValue();
1679             String dstVpnUuid = (nexthopIsVpn2) ? interVpnLink.getFirstEndpoint().getVpnUuid().getValue()
1680                     : interVpnLink.getSecondEndpoint().getVpnUuid().getValue();
1681             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
1682             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1683                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1684             if (newLabel == 0) {
1685                 LOG.error("Unable to fetch label from Id Manager. Bailing out of adding intervpnlink route for destination {}", destination);
1686                 return;
1687             }
1688             InterVpnLinkUtil.leakRoute(dataBroker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel);
1689         } else {
1690             if (rd != null) {
1691                 addPrefixToBGP(rd, destination, nextHop, label, writeConfigTxn);
1692             } else {
1693                 // ### add FIB route directly
1694                 fibManager.addOrUpdateFibEntry(dataBroker, routerID, destination, Arrays.asList(nextHop), label, RouteOrigin.STATIC, writeConfigTxn);
1695             }
1696         }
1697         if (!writeOperTxnPresent) {
1698             writeOperTxn.submit();
1699         }
1700         if (!writeConfigTxnPresent) {
1701             writeConfigTxn.submit();
1702         }
1703     }
1704
1705     protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName, WriteTransaction writeConfigTxn) {
1706         Boolean writeConfigTxnPresent = true;
1707         if (writeConfigTxn == null) {
1708             writeConfigTxnPresent = false;
1709             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1710         }
1711         if (intfName != null && !intfName.isEmpty()) {
1712             BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1713             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1714             if (nextHopIp == null || nextHopIp.isEmpty()) {
1715                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1716                         intfName, destination);
1717             }
1718             nextHop = nextHopIp;
1719         }
1720
1721         if (rd != null) {
1722             removePrefixFromBGP(rd, destination, nextHop, writeConfigTxn);
1723         } else {
1724             // ### add FIB route directly
1725             fibManager.removeOrUpdateFibEntry(dataBroker, routerID, destination, nextHop, writeConfigTxn);
1726         }
1727         if (!writeConfigTxnPresent) {
1728             writeConfigTxn.submit();
1729         }
1730     }
1731
1732     void publishAddNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1733         LOG.debug("Sending notification for add dpn {} in vpn {} event ", dpnId, vpnName);
1734         AddEventData data = new AddEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1735         AddDpnEvent event = new AddDpnEventBuilder().setAddEventData(data).build();
1736         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1737         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1738             @Override
1739             public void onFailure(Throwable error) {
1740                 LOG.warn("Error in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName, error);
1741             }
1742
1743             @Override
1744             public void onSuccess(Object arg) {
1745                 LOG.trace("Successful in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName);
1746             }
1747         });
1748     }
1749
1750     void publishRemoveNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1751         LOG.debug("Sending notification for remove dpn {} in vpn {} event ", dpnId, vpnName);
1752         RemoveEventData data = new RemoveEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1753         RemoveDpnEvent event = new RemoveDpnEventBuilder().setRemoveEventData(data).build();
1754         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1755         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1756             @Override
1757             public void onFailure(Throwable error) {
1758                 LOG.warn("Error in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName, error);
1759             }
1760
1761             @Override
1762             public void onSuccess(Object arg) {
1763                 LOG.trace("Successful in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName);
1764             }
1765         });
1766     }
1767
1768     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1769         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1770                 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1771                 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1772     }
1773
1774     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1775         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1776                 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1777     }
1778
1779     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1780         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1781         if(dpId.equals(BigInteger.ZERO)) {
1782             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
1783             return;
1784         }
1785         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1786
1787         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1788                 .OPERATIONAL, routerDpnListIdentifier);
1789         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1790         if (optionalRouterDpnList.isPresent()) {
1791             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1792                     RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1793         } else {
1794             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1795             builder.setRouterId(routerName);
1796             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1797             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
1798             routerInterfaces.add(routerInterface);
1799             builder.setDpnVpninterfacesList(Arrays.asList(dpnVpnList.build()));
1800             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1801                     getRouterId(routerName),
1802                     builder.build(), true);
1803         }
1804     }
1805
1806     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1807         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1808         if(dpId.equals(BigInteger.ZERO)) {
1809             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1810             return;
1811         }
1812         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1813         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1814                 .OPERATIONAL, routerDpnListIdentifier);
1815         if (optionalRouterDpnList.isPresent()) {
1816             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1817             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1818
1819             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1820                 if (routerInterfaces.isEmpty()) {
1821                     if (writeOperTxn != null) {
1822                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1823                     } else {
1824                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1825                     }
1826                 } else {
1827                     if (writeOperTxn != null) {
1828                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1829                                 RouterInterfaces.class,
1830                                 new RouterInterfacesKey(vpnInterfaceName)));
1831                     } else {
1832                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1833                                 RouterInterfaces.class,
1834                                 new RouterInterfacesKey(vpnInterfaceName)));
1835                     }
1836                 }
1837             }
1838         }
1839     }
1840
1841     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1842                                                   WriteTransaction writeOperTxn) {
1843         if(dpId.equals(BigInteger.ZERO)) {
1844             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1845             return;
1846         }
1847         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1848         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1849                 .OPERATIONAL, routerDpnListIdentifier);
1850         if (optionalRouterDpnList.isPresent()) {
1851             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1852             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1853             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1854                 if (routerInterfaces.isEmpty()) {
1855                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1856                 } else {
1857                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1858                             RouterInterfaces.class,
1859                             new RouterInterfacesKey(vpnInterfaceName)));
1860                 }
1861             }
1862         }
1863     }
1864
1865     //TODO(vivek) This waiting business to be removed in carbon
1866     public void waitForVpnInstance(String vpnName, long wait_time,
1867                                        ConcurrentHashMap<String, List<Runnable>> vpnInstanceMap) {
1868         List<Runnable> notifieeList = null;
1869         Runnable notifyTask = new VpnNotifyTask();
1870         try {
1871             synchronized (vpnInstanceMap) {
1872                 notifieeList = vpnInstanceMap.get(vpnName);
1873                 if (notifieeList == null) {
1874                     notifieeList = new ArrayList<Runnable>();
1875                     vpnInstanceMap.put(vpnName, notifieeList);
1876                 }
1877                 notifieeList.add(notifyTask);
1878             }
1879             synchronized (notifyTask) {
1880                 try {
1881                     notifyTask.wait(wait_time);
1882                 } catch (InterruptedException e) {
1883                 }
1884             }
1885         } finally {
1886             synchronized (vpnInstanceMap) {
1887                 notifieeList = vpnInstanceMap.get(vpnName);
1888                 if (notifieeList != null) {
1889                     notifieeList.remove(notifyTask);
1890                     if (notifieeList.isEmpty()) {
1891                         vpnInstanceMap.remove(vpnName);
1892                     }
1893                 }
1894             }
1895         }
1896     }
1897
1898     protected void createVpnInterfaceForRouter(VpnInterface vpnInterface, String interfaceName) {
1899         if (vpnInterface == null) {
1900             return;
1901         }
1902         String vpnName = vpnInterface.getVpnInstanceName();
1903         String rd = getRouteDistinguisher(vpnName);
1904         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
1905         if (adjs == null) {
1906             LOG.info("VPN Interface {} of router addition failed as adjacencies for "
1907                     + "this vpn interface could not be obtained", interfaceName);
1908             return;
1909         }
1910         if (rd == null || rd.isEmpty()) {
1911             rd = vpnName;
1912         }
1913         for (Adjacency adj : adjs) {
1914             if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
1915                 String primaryInterfaceIp = adj.getIpAddress();
1916                 String macAddress = adj.getMacAddress();
1917                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1918
1919                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1920                         VpnUtil.getNextHopLabelKey((rd == null) ? vpnName : rd, prefix));
1921
1922                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1923                         .setMacAddress(macAddress).setIpAddress(primaryInterfaceIp).build();
1924
1925                 VrfEntry vrfEntry = new VrfEntryBuilder().setKey(new VrfEntryKey(prefix)).setDestPrefix(prefix)
1926                         .setNextHopAddressList(Arrays.asList(primaryInterfaceIp)).setLabel(label)
1927                         .setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
1928                         .addAugmentation(RouterInterface.class, routerInt).build();
1929
1930                 List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1931                 InstanceIdentifierBuilder<VrfTables> idBuilder = InstanceIdentifier.builder(FibEntries.class)
1932                         .child(VrfTables.class, new VrfTablesKey(rd));
1933
1934                 InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1935                         .child(VrfTables.class, new VrfTablesKey(rd)).child(VrfEntry.class, new VrfEntryKey(prefix))
1936                         .build();
1937                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId, vrfEntry);
1938                 return;
1939             }
1940         }
1941         LOG.trace("VPN Interface {} of router addition failed as primary adjacency for"
1942                 + " this vpn interface could not be obtained", interfaceName);
1943     }
1944 }