vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Function;
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Preconditions;
  13 import com.google.common.base.Predicate;
  14 import com.google.common.base.Predicates;
  15 import com.google.common.collect.FluentIterable;
  16 import com.google.common.collect.Iterators;
  17 import com.google.common.util.concurrent.CheckedFuture;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.JdkFutureAdapters;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import java.math.BigInteger;
  23 import java.util.ArrayList;
  24 import java.util.Arrays;
  25 import java.util.Collection;
  26 import java.util.Iterator;
  27 import java.util.List;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ExecutionException;
  31 import java.util.concurrent.ExecutorService;
  32 import java.util.concurrent.Executors;
  33 import java.util.concurrent.Future;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  36 import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
  37 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  38 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  41 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  42 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  43 import org.opendaylight.genius.mdsalutil.ActionInfo;
  44 import org.opendaylight.genius.mdsalutil.ActionType;
  45 import org.opendaylight.genius.mdsalutil.FlowEntity;
  46 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  47 import org.opendaylight.genius.mdsalutil.InstructionType;
  48 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  49 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  50 import org.opendaylight.genius.mdsalutil.MatchInfo;
  51 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  52 import org.opendaylight.genius.mdsalutil.NwConstants;
  53 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  54 import org.opendaylight.genius.utils.ServiceIndex;
  55 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  56 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  57 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  58 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  59 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.VpnTargets;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInputBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEvent;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEventBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEvent;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEventBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventData;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventDataBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventData;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventDataBuilder;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 131 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 132 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 133 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 134 import org.opendaylight.yangtools.yang.common.RpcError;
 135 import org.opendaylight.yangtools.yang.common.RpcResult;
 136 import org.slf4j.Logger;
 137 import org.slf4j.LoggerFactory;
 138
 139 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
 140     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 141     private ListenerRegistration<DataChangeListener> listenerRegistration;
 142     private final DataBroker dataBroker;
 143     private final IBgpManager bgpManager;
 144     private final IFibManager fibManager;
 145     private final IMdsalApiManager mdsalManager;
 146     private final IdManagerService idManager;
 147     private final OdlArputilService arpManager;
 148     private final OdlInterfaceRpcService ifaceMgrRpcService;
 149     private final NotificationPublishService notificationPublishService;
 150     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
 151     private ExecutorService executorService = Executors.newSingleThreadExecutor();
 152
 153     public VpnInterfaceManager(final DataBroker dataBroker,
 154                                final IBgpManager bgpManager,
 155                                final OdlArputilService arpManager,
 156                                final IdManagerService idManager,
 157                                final IMdsalApiManager mdsalManager,
 158                                final IFibManager fibManager,
 159                                final OdlInterfaceRpcService ifaceMgrRpcService,
 160                                final NotificationPublishService notificationPublishService) {
 161         super(VpnInterface.class);
 162         this.dataBroker = dataBroker;
 163         this.bgpManager = bgpManager;
 164         this.arpManager = arpManager;
 165         this.idManager = idManager;
 166         this.mdsalManager = mdsalManager;
 167         this.fibManager = fibManager;
 168         this.ifaceMgrRpcService = ifaceMgrRpcService;
 169         this.notificationPublishService = notificationPublishService;
 170     }
 171
 172     public void start() {
 173         LOG.info("{} start", getClass().getSimpleName());
 174         listenerRegistration = dataBroker.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 175                 getWildCardPath(), this, DataChangeScope.SUBTREE);
 176     }
 177
 178     private InstanceIdentifier<VpnInterface> getWildCardPath() {
 179         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 180     }
 181
 182     @Override
 183     public void close() throws Exception {
 184         if (listenerRegistration != null) {
 185             listenerRegistration.close();
 186             listenerRegistration = null;
 187         }
 188         LOG.info("{} close", getClass().getSimpleName());
 189     }
 190
 191     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 192         return InstanceIdentifier.create(InterfacesState.class)
 193                 .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 194     }
 195
 196     @Override
 197     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 198         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 199         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 200         final String interfaceName = key.getName();
 201
 202         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 203                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 204         if(interfaceState != null){
 205             try{
 206                 final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 207                 final int ifIndex = interfaceState.getIfIndex();
 208                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 209                 dataStoreCoordinator.enqueueJob("VPNINTERFACE-"+ interfaceName,
 210                         new Callable<List<ListenableFuture<Void>>>() {
 211                             @Override
 212                             public List<ListenableFuture<Void>> call() throws Exception {
 213                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 214                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 215                                 WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 216                                 processVpnInterfaceUp(dpnId, vpnInterface, ifIndex, false, writeConfigTxn, writeOperTxn, writeInvTxn);
 217                                 CheckedFuture<Void, TransactionCommitFailedException> futures = writeOperTxn.submit();
 218                                 try {
 219                                     futures.get();
 220                                 } catch (InterruptedException | ExecutionException e) {
 221                                     LOG.error("Error adding oper data for interface {} to vpn {} on dpn {}", interfaceName,
 222                                             vpnInterface.getVpnInstanceName(), dpnId);
 223                                     throw new RuntimeException(e.getMessage());
 224                                 }
 225                                 futures = writeConfigTxn.submit();
 226                                 try {
 227                                     futures.get();
 228                                 } catch (InterruptedException | ExecutionException e) {
 229                                     LOG.error("Error adding config data for interface {} to vpn {} on dpn {}", interfaceName,
 230                                         vpnInterface.getVpnInstanceName(), dpnId);
 231                                     throw new RuntimeException(e.getMessage());
 232                                 }
 233                                 futures = writeInvTxn.submit();
 234                                 try {
 235                                     futures.get();
 236                                 } catch (InterruptedException | ExecutionException e) {
 237                                     LOG.error("Error adding inventory/flow data for interface {} to vpn {} on dpn {}", interfaceName,
 238                                         vpnInterface.getVpnInstanceName(), dpnId);
 239                                     throw new RuntimeException(e.getMessage());
 240                                 }
 241                                 return null;
 242                             }
 243                         });
 244             }catch (Exception e){
 245                 LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", interfaceName, e);
 246                 return;
 247             }
 248         } else {
 249             LOG.info("Handling addition of VPN interface {} skipped as interfaceState is not available", interfaceName);
 250         }
 251     }
 252
 253     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface,
 254                                          final int lPortTag, boolean isInterfaceUp,
 255                                          WriteTransaction writeConfigTxn,
 256                                          WriteTransaction writeOperTxn,
 257                                          WriteTransaction writeInvTxn) {
 258
 259         final String interfaceName = vpnInterface.getName();
 260         if (!isInterfaceUp) {
 261             final String vpnName = vpnInterface.getVpnInstanceName();
 262             LOG.info("Binding vpn service to interface {} ", interfaceName);
 263             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 264             if (vpnId == VpnConstants.INVALID_ID) {
 265                 LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 266                 return;
 267             }
 268             boolean waitForVpnInterfaceOpRemoval = false;
 269             VpnInterface opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, vpnInterface.getName());
 270             if (opVpnInterface != null ) {
 271                 String opVpnName = opVpnInterface.getVpnInstanceName();
 272                 String primaryInterfaceIp = null;
 273                 if(opVpnName.equals(vpnName)) {
 274                     // Please check if the primary VRF Entry does not exist for VPNInterface
 275                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 276                     // back to back
 277                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 278                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 279                     if (adjs == null) {
 280                         LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained", interfaceName);
 281                         return;
 282                     }
 283                     for (Adjacency adj : adjs) {
 284                         if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
 285                             primaryInterfaceIp = adj.getIpAddress();
 286                             break;
 287                         }
 288                     }
 289                     if (primaryInterfaceIp == null) {
 290                         LOG.info("VPN Interface {} addition failed as primary adjacency "
 291                                 + "for this vpn interface could not be obtained", interfaceName);
 292                         return;
 293                     }
 294                     // Get the rd of the vpn instance
 295                     String rd = getRouteDistinguisher(opVpnName);
 296                     rd =  (rd == null) ? opVpnName : rd;
 297                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, rd, primaryInterfaceIp);
 298                     if (vrf != null) {
 299                         LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
 300                         return;
 301                     }
 302                     waitForVpnInterfaceOpRemoval = true;
 303                 } else {
 304                     LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
 305                             interfaceName, vpnName, opVpnName);
 306                 }
 307             }
 308             if (!waitForVpnInterfaceOpRemoval) {
 309                 // Add the VPNInterface and quit
 310                 updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 311                 bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 312                 processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 313                 return;
 314             }
 315
 316             // FIB didn't get a chance yet to clean up this VPNInterface
 317             // Let us give it a chance here !
 318             LOG.info("VPN Interface {} waiting for FIB to clean up! ", interfaceName);
 319             try {
 320                 Runnable notifyTask = new VpnNotifyTask();
 321                 vpnIntfMap.put(interfaceName, notifyTask);
 322                 synchronized (notifyTask) {
 323                     try {
 324                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 325                     } catch (InterruptedException e) {
 326                     }
 327                 }
 328             } finally {
 329                 vpnIntfMap.remove(interfaceName);
 330             }
 331
 332             opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
 333             if (opVpnInterface != null) {
 334                 LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
 335                 return;
 336             }
 337             // VPNInterface got removed, proceed with Add
 338             updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 339             bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 340             processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 341         } else {
 342             // Interface is retained in the DPN, but its Link Up.
 343             // Advertise prefixes again for this interface to BGP
 344             advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
 345                     vpnInterface);
 346         }
 347     }
 348
 349
 350 //    private class UpdateDpnToVpnWorker implements Callable<List<ListenableFuture<Void>>> {
 351 //        BigInteger dpnId;
 352 //        String vpnName;
 353 //        String interfaceName;
 354 //        boolean addToDpn;
 355 //        int lPortTag;
 356 //
 357 //        public UpdateDpnToVpnWorker(BigInteger dpnId, String vpnName, String interfaceName,
 358 //                                    int lPortTag, boolean addToDpn) {
 359 //            this.dpnId= dpnId;
 360 //            this.vpnName = vpnName;
 361 //            this.interfaceName = interfaceName;
 362 //            this.lPortTag = lPortTag;
 363 //            this.addToDpn = addToDpn;
 364 //        }
 365 //
 366 //        @Override
 367 //        public List<ListenableFuture<Void>> call() throws Exception {
 368 //            // If another renderer(for eg : CSS) needs to be supported, check can be performed here
 369 //            // to call the respective helpers.
 370 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 371 //            updateDpnDbs(dpnId, vpnName, interfaceName, addToDpn, writeTxn);
 372 //            List<ListenableFuture<Void>> futures = new ArrayList<>();
 373 //            futures.add(writeTxn.submit());
 374 //            ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
 375 //            Futures.addCallback(listenableFuture,
 376 //                    new UpdateDpnToVpnCallback(dpnId, vpnName, interfaceName, lPortTag, addToDpn));
 377 //            return futures;
 378 //        }
 379 //    }
 380 //
 381 //
 382 //    /**
 383 //     * JobCallback class is used as a future callback for
 384 //     * main and rollback workers to handle success and failure.
 385 //     */
 386 //    private class UpdateDpnToVpnCallback implements FutureCallback<List<Void>> {
 387 //        BigInteger dpnId;
 388 //        String vpnName;
 389 //        String interfaceName;
 390 //        boolean addToDpn;
 391 //        int lPortTag;
 392 //
 393 //        public UpdateDpnToVpnCallback(BigInteger dpnId, String vpnName, String interfaceName,
 394 //                                      int lPortTag, boolean addToDpn) {
 395 //            this.dpnId= dpnId;
 396 //            this.vpnName = vpnName;
 397 //            this.interfaceName = interfaceName;
 398 //            this.lPortTag = lPortTag;
 399 //            this.addToDpn = addToDpn;
 400 //        }
 401 //
 402 //        /**
 403 //         * @param voids
 404 //         * This implies that all the future instances have returned success. -- TODO: Confirm this
 405 //         */
 406 //        @Override
 407 //        public void onSuccess(List<Void> voids) {
 408 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 409 //            bindService(dpnId, vpnName, interfaceName, lPortTag, writeTxn);
 410 //            processVpnInterfaceAdjacencies(dpnId, vpnName, interfaceName, writeTxn);
 411 //            writeTxn.submit();
 412 //        }
 413 //
 414 //        /**
 415 //         *
 416 //         * @param throwable
 417 //         * This method is used to handle failure callbacks.
 418 //         * If more retry needed, the retrycount is decremented and mainworker is executed again.
 419 //         * After retries completed, rollbackworker is executed.
 420 //         * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 421 //         */
 422 //
 423 //        @Override
 424 //        public void onFailure(Throwable throwable) {
 425 //            LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 426 //        }
 427 //    }
 428
 429
 430     private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
 431                                                  VpnInterface intf) {
 432         //Read NextHops
 433         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 434         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 435
 436         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 437         if (rd == null) {
 438             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 439                     intf.getName(), intf.getVpnInstanceName());
 440             return;
 441         } else {
 442             if (rd.equals(intf.getVpnInstanceName())) {
 443                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in " +
 444                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 445
 446                 return;
 447             }
 448         }
 449         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
 450                 intf.getVpnInstanceName(), rd);
 451
 452         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 453         if (nextHopIp == null){
 454             LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
 455             return;
 456         }
 457
 458         if (adjacencies.isPresent()) {
 459             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 460
 461             if (!nextHops.isEmpty()) {
 462                 LOG.trace("NextHops are " + nextHops);
 463                 for (Adjacency nextHop : nextHops) {
 464                     long label = nextHop.getLabel();
 465                     try {
 466                         LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 467                         bgpManager.advertisePrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
 468                         LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 469                     } catch(Exception e) {
 470                         LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
 471                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 472                     }
 473                 }
 474             }
 475         }
 476     }
 477
 478     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 479         //Read NextHops
 480         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 481         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 482
 483         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 484         if (rd == null) {
 485             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 486                     intf.getName(), intf.getVpnInstanceName());
 487             return;
 488         } else {
 489             if (rd.equals(intf.getVpnInstanceName())) {
 490                 LOG.info("withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in " +
 491                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 492                 return;
 493             }
 494         }
 495         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 496                 intf.getVpnInstanceName(), rd);
 497         if (adjacencies.isPresent()) {
 498             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 499
 500             if (!nextHops.isEmpty()) {
 501                 LOG.trace("NextHops are " + nextHops);
 502                 for (Adjacency nextHop : nextHops) {
 503                     try {
 504                         LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 505                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 506                         LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 507                     } catch(Exception e) {
 508                         LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
 509                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 510                     }
 511                 }
 512             }
 513         }
 514     }
 515
 516     public void updateVpnToDpnMapping(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 517         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 518         if (dpId == null) {
 519             dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, interfaceName);
 520         }
 521         if(!dpId.equals(BigInteger.ZERO)) {
 522             if(add)
 523                 createOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 524             else
 525                 removeOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 526         }
 527     }
 528
 529     private void bindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName, int lPortTag,
 530                              WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
 531         int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 532         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
 533
 534         int instructionKey = 0;
 535         List<Instruction> instructions = new ArrayList<Instruction>();
 536
 537         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(MetaDataUtil.getVpnIdMetadata(vpnId),
 538                 MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 539         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 540
 541         short l3vpn_service_index = ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX);
 542         BoundServices
 543                 serviceInfo =
 544                 InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 545                         l3vpn_service_index, priority,
 546                         NwConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 547         writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION,
 548                 InterfaceUtils.buildServiceId(vpnInterfaceName, l3vpn_service_index), serviceInfo, true);
 549         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
 550                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW, writeInvTxn);
 551         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
 552                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.ADD_FLOW, writeInvTxn);
 553
 554     }
 555
 556     private void processVpnInterfaceAdjacencies(BigInteger dpnId, String vpnName, String interfaceName,
 557                                                 WriteTransaction writeConfigTxn,
 558                                                 WriteTransaction writeOperTxn) {
 559         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 560         // Read NextHops
 561         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 562         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 563
 564         if (adjacencies.isPresent()) {
 565             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 566             List<Adjacency> value = new ArrayList<>();
 567
 568             // Get the rd of the vpn instance
 569             String rd = getRouteDistinguisher(vpnName);
 570
 571             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 572             if (nextHopIp == null){
 573                 LOG.error("NextHop for interface {} is null", interfaceName);
 574                 return;
 575             }
 576
 577             List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
 578
 579             LOG.trace("NextHops for interface {} are {}", interfaceName, nextHops);
 580             for (Adjacency nextHop : nextHops) {
 581                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 582                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 583                         VpnUtil.getNextHopLabelKey((rd == null) ? vpnName
 584                                 : rd, prefix));
 585                 List<String> adjNextHop = nextHop.getNextHopIpList();
 586                 value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIpList(
 587                         (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : Arrays.asList(nextHopIp))
 588                         .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 589
 590                 if (nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 591                     LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, interfaceName, vpnName);
 592                     writeOperTxn.merge(
 593                             LogicalDatastoreType.OPERATIONAL,
 594                             VpnUtil.getPrefixToInterfaceIdentifier(
 595                                     VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 596                             VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix), true);
 597                 } else {
 598                     //Extra route adjacency
 599                     LOG.trace("Adding prefix {} and nexthopList {} as extra-route for vpn", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName);
 600                     writeOperTxn.merge(
 601                             LogicalDatastoreType.OPERATIONAL,
 602                             VpnUtil.getVpnToExtrarouteIdentifier(
 603                                     (rd != null) ? rd : vpnName, nextHop.getIpAddress()),
 604                             VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()), true);
 605                 }
 606             }
 607
 608             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 609
 610             VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 611             InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 612             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, true);
 613             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 614
 615             for (Adjacency nextHop : aug.getAdjacency()) {
 616                 long label = nextHop.getLabel();
 617                 List<String> nextHopList = new ArrayList<>(nextHop.getNextHopIpList());
 618                 if (rd != null) {
 619                     addToLabelMapper(label, dpnId, nextHop.getIpAddress(), nextHopList, vpnId,
 620                             interfaceName, null,false, rd, writeOperTxn);
 621                     addPrefixToBGP(rd, nextHop.getIpAddress(), nextHopIp, label, writeConfigTxn);
 622                     //TODO: ERT - check for VPNs importing my route
 623                     for (VpnInstance vpn : vpnsToImportRoute) {
 624                         String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
 625                         if (vpnRd != null) {
 626                             LOG.debug("Exporting route with rd {} prefix {} nexthop {} label {} to VPN {}", vpnRd, nextHop.getIpAddress(), nextHopIp, label, vpn);
 627                             fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), Arrays.asList(nextHopIp), (int) label,
 628                                     RouteOrigin.SELF_IMPORTED, writeConfigTxn);
 629                         }
 630                     }
 631                 } else {
 632                     // ### add FIB route directly
 633                     fibManager.addOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), Arrays.asList(nextHopIp),
 634                             (int) label, RouteOrigin.STATIC, writeConfigTxn);
 635                 }
 636             }
 637         }
 638     }
 639
 640     private List<VpnInstance> getVpnsImportingMyRoute(final String vpnName) {
 641         List<VpnInstance> vpnsToImportRoute = new ArrayList<>();
 642
 643         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 644                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 645         Optional<VpnInstance> optVpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 646         final VpnInstance vpnInstance;
 647         if (optVpnInstance.isPresent()) {
 648             vpnInstance = optVpnInstance.get();
 649         } else {
 650             LOG.debug("Could not retrieve vpn instance {} to check for vpns importing the routes", vpnName);
 651             return vpnsToImportRoute;
 652         }
 653
 654         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 655             @Override
 656             public boolean apply(VpnInstance input) {
 657                 return !input.getVpnInstanceName().equals(vpnName);
 658             }
 659         };
 660
 661         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 662             @Override
 663             public boolean apply(VpnInstance input) {
 664                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ExportExtcommunity),
 665                         getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 666                 return Iterators.size(commonRTs.iterator()) > 0;
 667             }
 668         };
 669
 670         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 671             @Override
 672             public String apply(VpnInstance vpnInstance) {
 673                 //return vpnInstance.getVpnInstanceName();
 674                 return vpnInstance.getIpv4Family().getRouteDistinguisher();
 675             }
 676         };
 677
 678         vpnsToImportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(dataBroker)).
 679                 filter(excludeVpn).
 680                 filter(matchRTs).toList();
 681         return vpnsToImportRoute;
 682     }
 683
 684     private List<VpnInstance> getVpnsExportingMyRoute(final String vpnName) {
 685         List<VpnInstance> vpnsToExportRoute = new ArrayList<>();
 686
 687         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 688                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 689         Optional<VpnInstance> optVpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 690         final VpnInstance vpnInstance;
 691         if (optVpnInstance.isPresent()) {
 692             vpnInstance = optVpnInstance.get();
 693         } else {
 694             LOG.debug("Could not retrieve vpn instance {} to check for vpns exporting the routes", vpnName);
 695             return vpnsToExportRoute;
 696         }
 697
 698         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 699             @Override
 700             public boolean apply(VpnInstance input) {
 701                 return !input.getVpnInstanceName().equals(vpnName);
 702             }
 703         };
 704
 705         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 706             @Override
 707             public boolean apply(VpnInstance input) {
 708                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ImportExtcommunity),
 709                         getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
 710                 return Iterators.size(commonRTs.iterator()) > 0;
 711             }
 712         };
 713
 714         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 715             @Override
 716             public String apply(VpnInstance vpnInstance) {
 717                 return vpnInstance.getVpnInstanceName();
 718             }
 719         };
 720
 721         vpnsToExportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(dataBroker)).
 722                 filter(excludeVpn).
 723                 filter(matchRTs).toList();
 724         return vpnsToExportRoute;
 725     }
 726
 727     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
 728         final Predicate<T> inPredicate = Predicates.<T>in(collection2);
 729         return new Iterable<T>() {
 730             @Override
 731             public Iterator<T> iterator() {
 732                 return Iterators.filter(collection1.iterator(), inPredicate);
 733             }
 734         };
 735     }
 736
 737     private List<String> getRts(VpnInstance vpnInstance, VpnTarget.VrfRTType rtType) {
 738         String name = vpnInstance.getVpnInstanceName();
 739         List<String> rts = new ArrayList<>();
 740         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 741         if (vpnConfig == null) {
 742             LOG.trace("vpn config is not available for {}", name);
 743             return rts;
 744         }
 745         VpnTargets targets = vpnConfig.getVpnTargets();
 746         if (targets == null) {
 747             LOG.trace("vpn targets not available for {}", name);
 748             return rts;
 749         }
 750         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 751         if (vpnTargets == null) {
 752             LOG.trace("vpnTarget values not available for {}", name);
 753             return rts;
 754         }
 755         for (VpnTarget target : vpnTargets) {
 756             //TODO: Check for RT type is Both
 757             if(target.getVrfRTType().equals(rtType) ||
 758                     target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
 759                 String rtValue = target.getVrfRTValue();
 760                 rts.add(rtValue);
 761             }
 762         }
 763         return rts;
 764     }
 765
 766     private List<String> getExportRts(VpnInstance vpnInstance) {
 767         List<String> exportRts = new ArrayList<>();
 768         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 769         VpnTargets targets = vpnConfig.getVpnTargets();
 770         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 771         for (VpnTarget target : vpnTargets) {
 772             if (target.getVrfRTType().equals(VpnTarget.VrfRTType.ExportExtcommunity)) {
 773                 String rtValue = target.getVrfRTValue();
 774                 exportRts.add(rtValue);
 775             }
 776         }
 777         return exportRts;
 778     }
 779
 780     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 781                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow,
 782                              WriteTransaction writeConfigTxn){
 783         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 784         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, MetaDataUtil.getVpnIdMetadata(vpnId));
 785         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 786                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 787
 788         // Matching Arp reply flows
 789         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 790         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 791                 metadata, metadataMask }));
 792
 793         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 794
 795         // Instruction to punt to controller
 796         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 797         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 798         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 799         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
 800                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
 801
 802         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 803
 804         // Install the flow entry in L3_INTERFACE_TABLE
 805         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 806                 NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 807         FlowEntity flowEntity;
 808         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 809                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 810                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 811
 812         Flow flow = flowEntity.getFlowBuilder().build();
 813         String flowId = flowEntity.getFlowId();
 814         FlowKey flowKey = new FlowKey( new FlowId(flowId));
 815         Node nodeDpn = buildDpnNode(dpId);
 816
 817         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 818                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 819                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 820
 821         if (writeConfigTxn != null) {
 822             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 823                 LOG.debug("Creating ARP Flow for interface {}", vpnInterfaceName);
 824                 writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
 825             } else {
 826                 LOG.debug("Deleting ARP Flow for interface {}", vpnInterfaceName);
 827                 writeConfigTxn.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 828             }
 829         } else {
 830             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 831                 LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 832                 mdsalManager.installFlow(flowEntity);
 833             } else {
 834                 LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 835                 mdsalManager.removeFlow(flowEntity);
 836             }
 837         }
 838     }
 839
 840     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
 841     private Node buildDpnNode(BigInteger dpnId) {
 842         NodeId nodeId = new NodeId("openflow:" + dpnId);
 843         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
 844
 845         return nodeDpn;
 846     }
 847
 848     private String getRouteDistinguisher(String vpnName) {
 849         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 850                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 851         Optional<VpnInstance> vpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 852         String rd = "";
 853         if(vpnInstance.isPresent()) {
 854             VpnInstance instance = vpnInstance.get();
 855             VpnAfConfig config = instance.getIpv4Family();
 856             rd = config.getRouteDistinguisher();
 857         }
 858         return rd;
 859     }
 860
 861     private void createOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 862         String routeDistinguisher = getRouteDistinguisher(vpnName);
 863         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 864         Boolean newDpnOnVpn = Boolean.FALSE;
 865
 866         synchronized (vpnName.intern()) {
 867             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 868             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 869             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 870             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data
 871                     .entry.vpn.to.dpn.list.VpnInterfaces
 872                     vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 873
 874             if (dpnInVpn.isPresent()) {
 875                 if (writeTxn != null) {
 876                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id.child(
 877                             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance
 878                                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 879                             new VpnInterfacesKey(intfName)), vpnInterface, true);
 880                 } else {
 881                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
 882                             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance
 883                                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 884                             new VpnInterfacesKey(intfName)), vpnInterface);
 885                 }
 886             } else {
 887                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 888                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = new ArrayList<>();
 889                 vpnInterfaces.add(vpnInterface);
 890                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder().setDpnId(dpnId);
 891                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 892
 893                 if (writeTxn != null) {
 894                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 895                 } else {
 896                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 897                 }
 898                 newDpnOnVpn = Boolean.TRUE;
 899             }
 900             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
 901             try {
 902                 futures.get();
 903             } catch (InterruptedException | ExecutionException e) {
 904                 LOG.error("Error adding to dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
 905                 throw new RuntimeException(e.getMessage());
 906             }
 907              /*
 908             * Informing the Fib only after writeTxn is submitted successfuly.
 909             */
 910             if (newDpnOnVpn) {
 911                 LOG.debug("Sending populateFib event for new dpn {} in VPN {}", dpnId, vpnName);
 912                 fibManager.populateFibOnNewDpn(dpnId, vpnId, rd);
 913                 publishAddNotification(dpnId, vpnName, rd);
 914             }
 915         }
 916     }
 917
 918     private void removeOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 919         Boolean lastDpnOnVpn = Boolean.FALSE;
 920
 921         synchronized (vpnName.intern()) {
 922             String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
 923             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 924             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 925             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 926             if (dpnInVpn.isPresent()) {
 927                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 928                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 929                 org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 930                         currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 931
 932                 if (vpnInterfaces.remove(currVpnInterface)) {
 933                     if (vpnInterfaces.isEmpty()) {
 934                         List<IpAddresses> ipAddresses = dpnInVpn.get().getIpAddresses();
 935                         if (ipAddresses == null || ipAddresses.isEmpty()) {
 936                             VpnToDpnListBuilder dpnInVpnBuilder =
 937                                     new VpnToDpnListBuilder(dpnInVpn.get())
 938                                             .setDpnState(VpnToDpnList.DpnState.Inactive)
 939                                             .setVpnInterfaces(null);
 940                             if (writeTxn != null) {
 941                                 writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build(), true);
 942                             } else {
 943                                 VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build());
 944                             }
 945                             lastDpnOnVpn = Boolean.TRUE;
 946                         } else {
 947                             LOG.warn("vpn interfaces are empty but ip addresses are present for the vpn {} in dpn {}", vpnName, dpnId);
 948                         }
 949                     } else {
 950                         if (writeTxn != null) {
 951                             writeTxn.delete(LogicalDatastoreType.OPERATIONAL, id.child(
 952                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 953                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 954                                     new VpnInterfacesKey(intfName)));
 955                         } else {
 956                             VpnUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
 957                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 958                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 959                                     new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
 960                         }
 961                     }
 962                 }
 963             }
 964             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
 965             try {
 966                 futures.get();
 967             } catch (InterruptedException | ExecutionException e) {
 968                 LOG.error("Error removing from dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
 969                 throw new RuntimeException(e.getMessage());
 970             }
 971             if (lastDpnOnVpn) {
 972                 LOG.debug("Sending cleanup event for dpn {} in VPN {}", dpnId, vpnName);
 973                 fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
 974                 publishRemoveNotification(dpnId, vpnName, rd);
 975             }
 976         }
 977     }
 978
 979     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
 980         List<VpnInstance> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
 981         for (VpnInstance vpn : vpnsToExportRoute) {
 982             String rd = vpn.getIpv4Family().getRouteDistinguisher();
 983             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getIpv4Family().getRouteDistinguisher());
 984             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 985             if (vrfEntries != null) {
 986                 for (VrfEntry vrfEntry : vrfEntries) {
 987                     try {
 988                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
 989                             continue;
 990                         }
 991                         String prefix = vrfEntry.getDestPrefix();
 992                         long label = vrfEntry.getLabel();
 993                         List<String> nextHops = vrfEntry.getNextHopAddressList();
 994                         SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
 995                         for (String nh : nextHops) {
 996                             if (route != null) {
 997                                 LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
 998                                 importSubnetRouteForNewVpn(rd, prefix, nh, (int)label, route, writeConfigTxn);
 999                             } else {
1000                                 LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1001                                 fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, prefix, Arrays.asList(nh), (int)label,
1002                                         RouteOrigin.SELF_IMPORTED, writeConfigTxn);
1003                             }
1004                         }
1005                     } catch (Exception e) {
1006                         LOG.error("Exception occurred while importing route with prefix {} label {} nexthop {} from vpn {} to vpn {}", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpn.getVpnInstanceName(), vpnName);
1007                     }
1008                 }
1009                 writeConfigTxn.submit();
1010             } else {
1011                 LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getIpv4Family().getRouteDistinguisher());
1012             }
1013         }
1014     }
1015
1016     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label, WriteTransaction writeConfigTxn) {
1017         try {
1018             LOG.info("ADD: Adding Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1019             fibManager.addOrUpdateFibEntry(dataBroker, rd, prefix, Arrays.asList(nextHopIp), (int)label, RouteOrigin.STATIC, writeConfigTxn);
1020             bgpManager.advertisePrefix(rd, prefix, Arrays.asList(nextHopIp), (int)label);
1021             LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1022         } catch(Exception e) {
1023             LOG.error("Add prefix failed", e);
1024         }
1025     }
1026
1027     @Override
1028     public void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1029         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
1030         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1031         final String interfaceName = key.getName();
1032
1033         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1034         final Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1035         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
1036                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1037         if (optVpnInterface.isPresent()){
1038             BigInteger dpnId = BigInteger.ZERO;
1039             Boolean dpnIdRetrieved = Boolean.FALSE;
1040             if(interfaceState != null){
1041                 try{
1042                     dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1043                     dpnIdRetrieved = Boolean.TRUE;
1044                 }catch (Exception e){
1045                     LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", interfaceName, e);
1046                 }
1047             } else {
1048                 LOG.error("Unable to retrieve interfaceState for interface {} , quitting ", interfaceName);
1049                 return;
1050             }
1051             final VpnInterface vpnOpInterface = optVpnInterface.get();
1052             if(dpnIdRetrieved == Boolean.FALSE){
1053                 LOG.info("dpnId for {} has not been retrieved yet. Fetching from vpn interface operational DS", interfaceName);
1054                 dpnId = vpnOpInterface.getDpnId();
1055             }
1056             final int ifIndex = interfaceState.getIfIndex();
1057             final BigInteger dpId = dpnId;
1058             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1059             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1060                     new Callable<List<ListenableFuture<Void>>>() {
1061                         @Override
1062                         public List<ListenableFuture<Void>> call() throws Exception {
1063                             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1064                             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1065                             WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
1066                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, false, true, writeConfigTxn, writeOperTxn, writeInvTxn);
1067                             CheckedFuture<Void, TransactionCommitFailedException> futures = writeOperTxn.submit();
1068                             try {
1069                                 futures.get();
1070                             } catch (InterruptedException | ExecutionException e) {
1071                                 LOG.error("Error removing Oper data for interface {} from vpn {} on dpn {}", interfaceName,
1072                                         vpnOpInterface.getVpnInstanceName(), dpId);
1073                                 throw new RuntimeException(e.getMessage());
1074                             }
1075                             futures = writeConfigTxn.submit();
1076                             try {
1077                                 futures.get();
1078                             } catch (InterruptedException | ExecutionException e) {
1079                                 LOG.error("Error removing Config data for interface {} from vpn {} on dpn {}", interfaceName,
1080                                         vpnOpInterface.getVpnInstanceName(), dpId);
1081                                 throw new RuntimeException(e.getMessage());
1082                             }
1083                             futures = writeInvTxn.submit();
1084                             try {
1085                                 futures.get();
1086                             } catch (InterruptedException | ExecutionException e) {
1087                                 LOG.error("Error removing Inventory/Flow data for interface {} from vpn {} on dpn {}", interfaceName,
1088                                         vpnOpInterface.getVpnInstanceName(), dpId);
1089                                 throw new RuntimeException(e.getMessage());
1090                             }
1091                             return null;
1092                         }
1093                     });
1094
1095         }else{
1096             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
1097         }
1098     }
1099
1100     protected void processVpnInterfaceDown(BigInteger dpId,
1101                                            String interfaceName,
1102                                            int lPortTag,
1103                                            boolean isInterfaceStateDown,
1104                                            boolean isConfigRemoval,
1105                                            WriteTransaction writeConfigTxn,
1106                                            WriteTransaction writeOperTxn,
1107                                            WriteTransaction writeInvTxn) {
1108         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1109         if (!isInterfaceStateDown) {
1110             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1111             if(vpnInterface == null){
1112                 LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
1113                 return;
1114             }else{
1115                 final String vpnName = vpnInterface.getVpnInstanceName();
1116                 if(!vpnInterface.isScheduledForRemove()){
1117                     VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE, writeOperTxn);
1118                     removeAdjacenciesFromVpn(dpId, interfaceName, vpnInterface.getVpnInstanceName(), writeConfigTxn);
1119                     LOG.info("Unbinding vpn service from interface {} ", interfaceName);
1120                     unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown, isConfigRemoval, writeConfigTxn, writeInvTxn);
1121                 }else{
1122                     LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ", interfaceName);
1123                     return;
1124                 }
1125             }
1126         } else {
1127             // Interface is retained in the DPN, but its Link Down.
1128             // Only withdraw the prefixes for this interface from BGP
1129             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1130             if(vpnInterface == null){
1131                 LOG.info("Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in operational data store", interfaceName);
1132                 return;
1133             }else {
1134                 withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
1135             }
1136         }
1137     }
1138
1139     private void waitForFibToRemoveVpnPrefix(String interfaceName) {
1140         // FIB didn't get a chance yet to clean up this VPNInterface
1141         // Let us give it a chance here !
1142         LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
1143         try {
1144             Runnable notifyTask = new VpnNotifyTask();
1145             vpnIntfMap.put(interfaceName, notifyTask);
1146             synchronized (notifyTask) {
1147                 try {
1148                     notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
1149                 } catch (InterruptedException e) {
1150                 }
1151             }
1152         } finally {
1153             vpnIntfMap.remove(interfaceName);
1154         }
1155     }
1156
1157     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final String interfaceName, final String vpnName,
1158                                           WriteTransaction writeConfigTxn) {
1159         //Read NextHops
1160         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1161         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1162         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1163
1164         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1165         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", interfaceName,
1166                 vpnName, rd);
1167         if (adjacencies.isPresent()) {
1168             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1169
1170             if (!nextHops.isEmpty()) {
1171                 LOG.trace("NextHops are " + nextHops);
1172                 for (Adjacency nextHop : nextHops) {
1173                     List<String> nhList = new ArrayList<String>();
1174                     if (nextHop.getMacAddress() == null || nextHop.getMacAddress().isEmpty()) {
1175                         // This is either an extra-route (or) a learned IP via subnet-route
1176                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1177                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1178                             LOG.error("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
1179                                     rd, nextHop.getIpAddress());
1180                             continue;
1181                         }
1182                         nhList = Arrays.asList(nextHopIp);
1183                     } else {
1184                         // This is a primary adjacency
1185                         nhList = nextHop.getNextHopIpList();
1186                     }
1187                     if (rd.equals(vpnName)) {
1188                         //this is an internal vpn - the rd is assigned to the vpn instance name;
1189                         //remove from FIB directly
1190                         for(String nh : nhList) {
1191                             fibManager.removeOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), nh, writeConfigTxn);
1192                         }
1193                     } else {
1194                         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1195                         for (String nh : nhList) {
1196                             //IRT: remove routes from other vpns importing it
1197                             removePrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1198                             for (VpnInstance vpn : vpnsToImportRoute) {
1199                                 String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
1200                                 if (vpnRd != null) {
1201                                     LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd, nextHop.getIpAddress(), vpn.getVpnInstanceName());
1202                                     fibManager.removeOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), nh, writeConfigTxn);
1203                                 }
1204                             }
1205                         }
1206                     }
1207                     String ip = nextHop.getIpAddress().split("/")[0];
1208                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
1209                             vpnName, ip);
1210                     if (vpnPortipToPort != null && !vpnPortipToPort.isConfig()) {
1211                         LOG.trace("VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
1212                                 vpnPortipToPort.getPortName(),ip);
1213                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip);
1214                     }
1215                 }
1216             }
1217         }
1218     }
1219
1220
1221     private void unbindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName,
1222                                int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval,
1223                                WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
1224         short l3vpn_service_index = ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX);
1225         if (!isInterfaceStateDown && isConfigRemoval) {
1226             writeConfigTxn.delete(LogicalDatastoreType.CONFIGURATION,
1227                     InterfaceUtils.buildServiceId(vpnInterfaceName,
1228                             l3vpn_service_index));
1229         }
1230         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
1231         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
1232                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW, writeInvTxn);
1233         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
1234                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.DEL_FLOW, writeInvTxn);
1235     }
1236
1237
1238     private void removePrefixFromBGP(String rd, String prefix, String nextHop, WriteTransaction writeConfigTxn) {
1239         try {
1240             LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
1241             fibManager.removeOrUpdateFibEntry(dataBroker, rd, prefix, nextHop, writeConfigTxn);
1242             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1243             LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
1244         } catch(Exception e) {
1245             LOG.error("Delete prefix failed", e);
1246         }
1247     }
1248
1249     @Override
1250     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
1251         LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
1252         String oldVpnName = original.getVpnInstanceName();
1253         String newVpnName = update.getVpnInstanceName();
1254         BigInteger dpnId = update.getDpnId();
1255         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
1256         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
1257         if (oldAdjs == null) {
1258             oldAdjs = new ArrayList<>();
1259         }
1260         if (newAdjs == null) {
1261             newAdjs = new ArrayList<>();
1262         }
1263         //handles switching between <internal VPN - external VPN>
1264         if (!oldVpnName.equals(newVpnName)) {
1265             remove(identifier, original);
1266             waitForFibToRemoveVpnPrefix(update.getName());
1267             add(identifier, update);
1268         }
1269         //handle both addition and removal of adjacencies
1270         //currently, new adjacency may be an extra route
1271         if (!oldAdjs.equals(newAdjs)) {
1272             for (Adjacency adj : newAdjs) {
1273                 if (oldAdjs.contains(adj)) {
1274                     oldAdjs.remove(adj);
1275                 } else {
1276                     // add new adjacency - right now only extra route will hit this path
1277                     addNewAdjToVpnInterface(identifier, adj, dpnId);
1278                 }
1279             }
1280             for (Adjacency adj : oldAdjs) {
1281                 delAdjFromVpnInterface(identifier, adj, dpnId);
1282             }
1283         }
1284     }
1285
1286     public void processArpRequest(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715
1287                                           .IpAddress srcIP, PhysAddress srcMac, org.opendaylight.yang.gen.v1.urn.ietf
1288             .params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress targetIP, PhysAddress targetMac, String srcInterface){
1289         //Build ARP response with ARP requests TargetIp TargetMac as the Arp Response SrcIp and SrcMac
1290         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
1291                 .setDstIpaddress(srcIP).setDstMacaddress(srcMac).setSrcIpaddress(targetIP).setSrcMacaddress(targetMac).build();
1292         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
1293         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
1294         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
1295             @Override
1296             public void onFailure(Throwable error) {
1297                 LOG.error("Error - {}", msgFormat, error);
1298             }
1299
1300             @Override
1301             public void onSuccess(RpcResult<Void> result) {
1302                 if(!result.isSuccessful()) {
1303                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
1304                 } else {
1305                     LOG.debug("Successful RPC Result - {}", msgFormat);
1306                 }
1307             }
1308         });
1309     }
1310
1311     private String getErrorText(Collection<RpcError> errors) {
1312         StringBuilder errorText = new StringBuilder();
1313         for(RpcError error : errors) {
1314             errorText.append(",").append(error.getErrorType()).append("-")
1315                     .append(error.getMessage());
1316         }
1317         return errorText.toString();
1318     }
1319
1320     private void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1321                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd,
1322                                   WriteTransaction writeOperTxn) {
1323         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1324         Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
1325         Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
1326         Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
1327         if (!isSubnetRoute) {
1328             // NextHop must be present for non-subnetroute entries
1329             Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
1330         }
1331         LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
1332         if (dpnId != null) {
1333             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1334                     .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
1335             LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1336             lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(vpnId)
1337                     .setIsSubnetRoute(isSubnetRoute);
1338             if (elanTag != null) {
1339                 lriBuilder.setElanTag(elanTag);
1340             }
1341             if (vpnInterfaceName != null) {
1342                 lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1343             }
1344             lriBuilder.setParentVpnRd(rd);
1345             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
1346             if (vpnInstanceOpDataEntry != null) {
1347                 List<String> vpnInstanceNames = Arrays.asList(vpnInstanceOpDataEntry.getVpnInstanceName());
1348                 lriBuilder.setVpnInstanceList(vpnInstanceNames);
1349             }
1350             LabelRouteInfo lri = lriBuilder.build();
1351             LOG.trace("Adding route info to label map: {}", lri);
1352             if (writeOperTxn != null) {
1353                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, lriIid, lri, true);
1354             } else {
1355                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
1356             }
1357         } else {
1358             LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
1359         }
1360     }
1361
1362     public synchronized void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
1363                                                         long elantag, BigInteger dpnId, WriteTransaction writeTxn) {
1364         SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
1365         RouteOrigin origin = RouteOrigin.STATIC; // Only case when a route is considered as directly connected
1366         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1367                 .setLabel((long)label).setOrigin(origin.getValue())
1368                 .addAugmentation(SubnetRoute.class, route).build();
1369
1370         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
1371
1372         //TODO: What should be parentVpnId? Get it from RD?
1373         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1374         addToLabelMapper((long)label, dpnId, prefix, Arrays.asList(nextHop), vpnId, null, elantag, true, rd, null);
1375         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1376
1377         InstanceIdentifierBuilder<VrfTables> idBuilder =
1378                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1379         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1380
1381         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1382                 setVrfEntry(vrfEntryList).build();
1383
1384         if (writeTxn != null) {
1385             writeTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1386         } else {
1387             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1388         }
1389
1390         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1391         if (vpnsToImportRoute.size() > 0) {
1392             VrfEntry importingVrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1393                     .setLabel((long) label).setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
1394                     .addAugmentation(SubnetRoute.class, route).build();
1395             List<VrfEntry> importingVrfEntryList = Arrays.asList(importingVrfEntry);
1396             for (VpnInstance vpnInstance : vpnsToImportRoute) {
1397                 LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop, label, vpnInstance.getVpnInstanceName());
1398                 String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1399                 InstanceIdentifier<VrfTables> importingVrfTableId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(importingRd)).build();
1400                 VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(importingVrfEntryList).build();
1401                 if (writeTxn != null) {
1402                     writeTxn.merge(LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable, true);
1403                 } else {
1404                     VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable);
1405                 }
1406             }
1407         }
1408     }
1409
1410     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1411                                                         SubnetRoute route, WriteTransaction writeConfigTxn) {
1412
1413         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1414         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1415                 .setLabel((long)label).setOrigin(origin.getValue())
1416                 .addAugmentation(SubnetRoute.class, route).build();
1417         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, route.getElantag());
1418         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1419         InstanceIdentifierBuilder<VrfTables> idBuilder =
1420                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1421         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1422         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1423                 setVrfEntry(vrfEntryList).build();
1424         if (writeConfigTxn != null) {
1425             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1426         } else {
1427             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1428         }
1429     }
1430
1431     public synchronized void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName){
1432         fibManager.removeOrUpdateFibEntry(dataBroker, rd, prefix, null /* nextHopToRemove */, null);
1433         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1434         for (VpnInstance vpnInstance : vpnsToImportRoute) {
1435             String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1436             LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix, vpnInstance.getVpnInstanceName());
1437             fibManager.removeOrUpdateFibEntry(dataBroker, importingRd, prefix, null /* nextHopToRemove */, null);
1438         }
1439     }
1440
1441     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1442
1443         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1444
1445         if (optVpnInterface.isPresent()) {
1446             VpnInterface currVpnIntf = optVpnInterface.get();
1447             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1448             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1449             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1450             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, adjPath);
1451             long label =
1452                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1453                             VpnUtil.getNextHopLabelKey((rd != null) ? rd : currVpnIntf.getVpnInstanceName(), prefix));
1454
1455             List<Adjacency> adjacencies;
1456             if (optAdjacencies.isPresent()) {
1457                 adjacencies = optAdjacencies.get().getAdjacency();
1458             } else {
1459                 //This code will not be hit since VM adjacency will always be there
1460                 adjacencies = new ArrayList<>();
1461             }
1462
1463             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIpList(adj.getNextHopIpList())
1464                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
1465
1466             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1467             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove());
1468
1469             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1470             for (String nh : adj.getNextHopIpList()) {
1471                 addExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(), (int) label,
1472                         currVpnIntf.getName());
1473             }
1474         }
1475     }
1476
1477     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1478         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1479
1480         if (optVpnInterface.isPresent()) {
1481             VpnInterface currVpnIntf = optVpnInterface.get();
1482
1483             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1484             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1485             if (optAdjacencies.isPresent()) {
1486                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1487
1488                 if (!adjacencies.isEmpty()) {
1489                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1490                     LOG.trace("Adjacencies are " + adjacencies);
1491                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1492                     while (adjIt.hasNext()) {
1493                         Adjacency adjElem = adjIt.next();
1494                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1495                             adjIt.remove();
1496
1497                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1498                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1499                                     currVpnIntf.getVpnInstanceName(),
1500                                     aug, dpnId, currVpnIntf.isScheduledForRemove());
1501
1502                             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1503
1504                             for (String nh : adj.getNextHopIpList()) {
1505                                 delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
1506                                         currVpnIntf.getName());
1507                             }
1508                             break;
1509                         }
1510
1511                     }
1512                 }
1513             }
1514         }
1515
1516     }
1517
1518     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label,
1519                                  String intfName) {
1520
1521         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1522         VpnUtil.syncUpdate(
1523                 dataBroker,
1524                 LogicalDatastoreType.OPERATIONAL,
1525                 VpnUtil.getVpnToExtrarouteIdentifier( (rd != null) ? rd : routerID, destination),
1526                 VpnUtil.getVpnToExtraroute(destination, Arrays.asList(nextHop)));
1527
1528         BigInteger dpnId = null;
1529         if (intfName != null && !intfName.isEmpty()) {
1530             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1531             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1532             if (nextHopIp == null || nextHopIp.isEmpty()) {
1533                 LOG.error("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1534                         intfName, destination);
1535                 return;
1536             }
1537             nextHop = nextHopIp;
1538         }
1539         List<String> nextHopIpList = Arrays.asList(nextHop);
1540         if (rd != null) {
1541             /* Label mapper is required only for BGP VPN and not for Internal VPN */
1542             addToLabelMapper((long) label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(dataBroker, routerID),
1543                     intfName, null, false, rd, null);
1544         }
1545
1546         // TODO (eperefr): This is a limitation to be stated in docs. When configuring static route to go to
1547         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1548         // first place.
1549         InterVpnLink interVpnLink = VpnUtil.getInterVpnLinkByEndpointIp(dataBroker, nextHop);
1550         if ( interVpnLink != null ) {
1551             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1552             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1553             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1554             // This is like leaking one of the Vpn2 routes towards Vpn1
1555             boolean nexthopIsVpn2 = ( interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nextHop) );
1556             String srcVpnUuid = (nexthopIsVpn2) ? interVpnLink.getSecondEndpoint().getVpnUuid().getValue()
1557                     : interVpnLink.getFirstEndpoint().getVpnUuid().getValue();
1558             String dstVpnUuid = (nexthopIsVpn2) ? interVpnLink.getFirstEndpoint().getVpnUuid().getValue()
1559                     : interVpnLink.getSecondEndpoint().getVpnUuid().getValue();
1560             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
1561             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1562                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1563             VpnUtil.leakRoute(dataBroker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel);
1564         } else {
1565             if (rd != null) {
1566                 addPrefixToBGP(rd, destination, nextHop, label, null);
1567             } else {
1568                 // ### add FIB route directly
1569                 fibManager.addOrUpdateFibEntry(dataBroker, routerID, destination, Arrays.asList(nextHop), label, RouteOrigin.STATIC, null);
1570             }
1571         }
1572     }
1573
1574     protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName) {
1575         if (intfName != null && !intfName.isEmpty()) {
1576             BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1577             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1578             if (nextHopIp == null || nextHopIp.isEmpty()) {
1579                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1580                         intfName, destination);
1581             }
1582             nextHop = nextHopIp;
1583         }
1584
1585         if (rd != null) {
1586             removePrefixFromBGP(rd, destination, nextHop, null);
1587         } else {
1588             // ### add FIB route directly
1589             fibManager.removeOrUpdateFibEntry(dataBroker, routerID, destination, nextHop, null);
1590         }
1591     }
1592
1593     void publishAddNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1594         LOG.debug("Sending notification for add dpn {} in vpn {} event ", dpnId, vpnName);
1595         AddEventData data = new AddEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1596         AddDpnEvent event = new AddDpnEventBuilder().setAddEventData(data).build();
1597         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1598         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1599             @Override
1600             public void onFailure(Throwable error) {
1601                 LOG.warn("Error in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName, error);
1602             }
1603
1604             @Override
1605             public void onSuccess(Object arg) {
1606                 LOG.trace("Successful in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName);
1607             }
1608         });
1609     }
1610
1611     void publishRemoveNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1612         LOG.debug("Sending notification for remove dpn {} in vpn {} event ", dpnId, vpnName);
1613         RemoveEventData data = new RemoveEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1614         RemoveDpnEvent event = new RemoveDpnEventBuilder().setRemoveEventData(data).build();
1615         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1616         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1617             @Override
1618             public void onFailure(Throwable error) {
1619                 LOG.warn("Error in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName, error);
1620             }
1621
1622             @Override
1623             public void onSuccess(Object arg) {
1624                 LOG.trace("Successful in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName);
1625             }
1626         });
1627     }
1628
1629     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1630         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1631                 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1632                 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1633     }
1634
1635     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1636         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1637                 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1638     }
1639
1640     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1641         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1642         if(dpId.equals(BigInteger.ZERO)) {
1643             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
1644             return;
1645         }
1646         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1647
1648         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1649                 .OPERATIONAL, routerDpnListIdentifier);
1650         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1651         if (optionalRouterDpnList.isPresent()) {
1652             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1653                     RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1654         } else {
1655             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1656             builder.setRouterId(routerName);
1657             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1658             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
1659             routerInterfaces.add(routerInterface);
1660             builder.setDpnVpninterfacesList(Arrays.asList(dpnVpnList.build()));
1661             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1662                     getRouterId(routerName),
1663                     builder.build(), true);
1664         }
1665     }
1666
1667     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1668         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1669         if(dpId.equals(BigInteger.ZERO)) {
1670             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1671             return;
1672         }
1673         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1674         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1675                 .OPERATIONAL, routerDpnListIdentifier);
1676         if (optionalRouterDpnList.isPresent()) {
1677             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1678             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1679
1680             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1681                 if (routerInterfaces.isEmpty()) {
1682                     if (writeOperTxn != null) {
1683                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1684                     } else {
1685                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1686                     }
1687                 } else {
1688                     if (writeOperTxn != null) {
1689                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1690                                 RouterInterfaces.class,
1691                                 new RouterInterfacesKey(vpnInterfaceName)));
1692                     } else {
1693                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1694                                 RouterInterfaces.class,
1695                                 new RouterInterfacesKey(vpnInterfaceName)));
1696                     }
1697                 }
1698             }
1699         }
1700     }
1701
1702     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1703                                                   WriteTransaction writeOperTxn) {
1704         if(dpId.equals(BigInteger.ZERO)) {
1705             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1706             return;
1707         }
1708         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1709         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1710                 .OPERATIONAL, routerDpnListIdentifier);
1711         if (optionalRouterDpnList.isPresent()) {
1712             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1713             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1714             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1715                 if (routerInterfaces.isEmpty()) {
1716                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1717                 } else {
1718                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1719                             RouterInterfaces.class,
1720                             new RouterInterfacesKey(vpnInterfaceName)));
1721                 }
1722             }
1723         }
1724     }
1725
1726     public void addMIPAdjacency(String vpnName,String vpnInterface, org.opendaylight.yang.gen.v1.urn.ietf.params.xml
1727             .ns.yang.ietf.inet.types.rev130715.IpAddress prefix){
1728
1729         LOG.trace("Adding {} adjacency to VPN Interface {} ",prefix,vpnInterface);
1730         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
1731         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
1732         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
1733         String nextHopIpAddr = null;
1734         String nextHopMacAddress = null;
1735         String ip = prefix.getIpv4Address().getValue();
1736         if (adjacencies.isPresent()) {
1737             List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
1738             ip = VpnUtil.getIpPrefix(ip);
1739             for (Adjacency adjacs : adjacencyList) {
1740                 if (adjacs.getMacAddress() != null && !adjacs.getMacAddress().isEmpty()) {
1741                     nextHopIpAddr = adjacs.getIpAddress();
1742                     nextHopMacAddress = adjacs.getMacAddress();
1743                     break;
1744                 }
1745             }
1746             if (nextHopMacAddress != null && ip != null) {
1747                 String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1748                 long label =
1749                         VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1750                                 VpnUtil.getNextHopLabelKey((rd != null) ? rd : vpnName, ip));
1751                 String nextHopIp = nextHopIpAddr.split("/")[0];
1752                 Adjacency newAdj = new AdjacencyBuilder().setIpAddress(ip).setKey
1753                         (new AdjacencyKey(ip)).setNextHopIpList(Arrays.asList(nextHopIp)).build();
1754                 adjacencyList.add(newAdj);
1755                 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
1756                 VpnInterface newVpnIntf = new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(vpnInterface)).
1757                         setName(vpnInterface).setVpnInstanceName(vpnName).addAugmentation(Adjacencies.class, aug).build();
1758                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vpnIfId, newVpnIntf);
1759                 LOG.debug(" Successfully stored subnetroute Adjacency into VpnInterface {}", vpnInterface);
1760             }
1761         }
1762
1763     }
1764
1765     public void removeMIPAdjacency(String vpnName, String vpnInterface, org.opendaylight.yang.gen.v1.urn.ietf.params
1766             .xml.ns.yang.ietf.inet.types.rev130715.IpAddress prefix) {
1767         String ip = VpnUtil.getIpPrefix(prefix.getIpv4Address().getValue());
1768         LOG.trace("Removing {} adjacency from Old VPN Interface {} ",ip,vpnInterface);
1769         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
1770         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
1771         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1772         if (adjacencies.isPresent()) {
1773             InstanceIdentifier<Adjacency> adjacencyIdentifier = InstanceIdentifier.builder(VpnInterfaces.class).
1774                     child(VpnInterface.class, new VpnInterfaceKey(vpnInterface)).augmentation(Adjacencies.class)
1775                     .child(Adjacency.class, new AdjacencyKey(ip)).build();
1776             MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, adjacencyIdentifier);
1777             LOG.trace("Successfully Deleted Adjacency into VpnInterface {}", vpnInterface);
1778         }
1779     }
1780
1781     void notifyTaskIfRequired(String intfName) {
1782         Runnable notifyTask = vpnIntfMap.remove(intfName);
1783         if (notifyTask == null) {
1784             return;
1785         }
1786         executorService.execute(notifyTask);
1787     }
1788 }