vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Function;
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Preconditions;
  13 import com.google.common.base.Predicate;
  14 import com.google.common.base.Predicates;
  15 import com.google.common.collect.FluentIterable;
  16 import com.google.common.collect.Iterators;
  17 import com.google.common.util.concurrent.CheckedFuture;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.JdkFutureAdapters;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import java.math.BigInteger;
  23 import java.util.ArrayList;
  24 import java.util.Arrays;
  25 import java.util.Collection;
  26 import java.util.Iterator;
  27 import java.util.List;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ExecutionException;
  31 import java.util.concurrent.ExecutorService;
  32 import java.util.concurrent.Executors;
  33 import java.util.concurrent.Future;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  36 import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
  37 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  38 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  41 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  42 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  43 import org.opendaylight.genius.mdsalutil.ActionInfo;
  44 import org.opendaylight.genius.mdsalutil.ActionType;
  45 import org.opendaylight.genius.mdsalutil.FlowEntity;
  46 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  47 import org.opendaylight.genius.mdsalutil.InstructionType;
  48 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  49 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  50 import org.opendaylight.genius.mdsalutil.MatchInfo;
  51 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  52 import org.opendaylight.genius.mdsalutil.NwConstants;
  53 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  54 import org.opendaylight.genius.utils.ServiceIndex;
  55 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  56 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  57 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  58 import org.opendaylight.netvirt.vpnmanager.intervpnlink.InterVpnLinkUtil;
  59 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.VpnTargets;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  71 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInput;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInputBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEvent;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEventBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEvent;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEventBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventData;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventDataBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventData;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventDataBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 133 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 134 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 135 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 136 import org.opendaylight.yangtools.yang.common.RpcError;
 137 import org.opendaylight.yangtools.yang.common.RpcResult;
 138 import org.slf4j.Logger;
 139 import org.slf4j.LoggerFactory;
 140
 141 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
 142     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 143     private ListenerRegistration<DataChangeListener> listenerRegistration;
 144     private final DataBroker dataBroker;
 145     private final IBgpManager bgpManager;
 146     private final IFibManager fibManager;
 147     private final IMdsalApiManager mdsalManager;
 148     private final IdManagerService idManager;
 149     private final OdlArputilService arpManager;
 150     private final OdlInterfaceRpcService ifaceMgrRpcService;
 151     private final NotificationPublishService notificationPublishService;
 152     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
 153     private ExecutorService executorService = Executors.newSingleThreadExecutor();
 154
 155     public VpnInterfaceManager(final DataBroker dataBroker,
 156                                final IBgpManager bgpManager,
 157                                final OdlArputilService arpManager,
 158                                final IdManagerService idManager,
 159                                final IMdsalApiManager mdsalManager,
 160                                final IFibManager fibManager,
 161                                final OdlInterfaceRpcService ifaceMgrRpcService,
 162                                final NotificationPublishService notificationPublishService) {
 163         super(VpnInterface.class);
 164         this.dataBroker = dataBroker;
 165         this.bgpManager = bgpManager;
 166         this.arpManager = arpManager;
 167         this.idManager = idManager;
 168         this.mdsalManager = mdsalManager;
 169         this.fibManager = fibManager;
 170         this.ifaceMgrRpcService = ifaceMgrRpcService;
 171         this.notificationPublishService = notificationPublishService;
 172     }
 173
 174     public void start() {
 175         LOG.info("{} start", getClass().getSimpleName());
 176         listenerRegistration = dataBroker.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 177                 getWildCardPath(), this, DataChangeScope.SUBTREE);
 178     }
 179
 180     private InstanceIdentifier<VpnInterface> getWildCardPath() {
 181         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 182     }
 183
 184     @Override
 185     public void close() throws Exception {
 186         if (listenerRegistration != null) {
 187             listenerRegistration.close();
 188             listenerRegistration = null;
 189         }
 190         LOG.info("{} close", getClass().getSimpleName());
 191     }
 192
 193     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 194         return InstanceIdentifier.create(InterfacesState.class)
 195                 .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 196     }
 197
 198     @Override
 199     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 200         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 201         LOG.info("VPN Interface add event - intfName {}" ,vpnInterface.getName());
 202         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 203         final String interfaceName = key.getName();
 204
 205         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 206                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 207         if(interfaceState != null){
 208             try{
 209                 final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 210                 final int ifIndex = interfaceState.getIfIndex();
 211                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 212                 dataStoreCoordinator.enqueueJob("VPNINTERFACE-"+ interfaceName,
 213                         new Callable<List<ListenableFuture<Void>>>() {
 214                             @Override
 215                             public List<ListenableFuture<Void>> call() throws Exception {
 216                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 217                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 218                                 WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 219                                 processVpnInterfaceUp(dpnId, vpnInterface, ifIndex, false, writeConfigTxn, writeOperTxn, writeInvTxn);
 220                                 List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
 221                                 futures.add(writeOperTxn.submit());
 222                                 futures.add(writeConfigTxn.submit());
 223                                 futures.add(writeInvTxn.submit());
 224                                 return futures;
 225                             }
 226                         });
 227             }catch (Exception e){
 228                 LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", interfaceName, e);
 229                 return;
 230             }
 231         } else {
 232             LOG.info("Handling addition of VPN interface {} skipped as interfaceState is not available", interfaceName);
 233         }
 234     }
 235
 236     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface,
 237                                          final int lPortTag, boolean isInterfaceUp,
 238                                          WriteTransaction writeConfigTxn,
 239                                          WriteTransaction writeOperTxn,
 240                                          WriteTransaction writeInvTxn) {
 241
 242         final String interfaceName = vpnInterface.getName();
 243         if (!isInterfaceUp) {
 244             final String vpnName = vpnInterface.getVpnInstanceName();
 245             LOG.info("Binding vpn service to interface {} ", interfaceName);
 246             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 247             if (vpnId == VpnConstants.INVALID_ID) {
 248                 LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 249                 return;
 250             }
 251             boolean waitForVpnInterfaceOpRemoval = false;
 252             VpnInterface opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, vpnInterface.getName());
 253             if (opVpnInterface != null ) {
 254                 String opVpnName = opVpnInterface.getVpnInstanceName();
 255                 String primaryInterfaceIp = null;
 256                 if(opVpnName.equals(vpnName)) {
 257                     // Please check if the primary VRF Entry does not exist for VPNInterface
 258                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 259                     // back to back
 260                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 261                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 262                     if (adjs == null) {
 263                         LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained", interfaceName);
 264                         return;
 265                     }
 266                     for (Adjacency adj : adjs) {
 267                         if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
 268                             primaryInterfaceIp = adj.getIpAddress();
 269                             break;
 270                         }
 271                     }
 272                     if (primaryInterfaceIp == null) {
 273                         LOG.info("VPN Interface {} addition failed as primary adjacency "
 274                                 + "for this vpn interface could not be obtained", interfaceName);
 275                         return;
 276                     }
 277                     // Get the rd of the vpn instance
 278                     String rd = getRouteDistinguisher(opVpnName);
 279                     rd =  (rd == null) ? opVpnName : rd;
 280                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, rd, primaryInterfaceIp);
 281                     if (vrf != null) {
 282                         LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
 283                         return;
 284                     }
 285                     waitForVpnInterfaceOpRemoval = true;
 286                 } else {
 287                     LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
 288                             interfaceName, vpnName, opVpnName);
 289                 }
 290             }
 291             if (!waitForVpnInterfaceOpRemoval) {
 292                 // Add the VPNInterface and quit
 293                 updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 294                 bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 295                 processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 296                 return;
 297             }
 298
 299             // FIB didn't get a chance yet to clean up this VPNInterface
 300             // Let us give it a chance here !
 301             LOG.info("Trying to add VPN Interface {}, but waiting for FIB to clean up! ", interfaceName);
 302             try {
 303                 Runnable notifyTask = new VpnNotifyTask();
 304                 vpnIntfMap.put(interfaceName, notifyTask);
 305                 synchronized (notifyTask) {
 306                     try {
 307                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 308                     } catch (InterruptedException e) {
 309                     }
 310                 }
 311             } finally {
 312                 vpnIntfMap.remove(interfaceName);
 313             }
 314
 315             opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
 316             if (opVpnInterface != null) {
 317                 LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
 318                 return;
 319             }
 320             // VPNInterface got removed, proceed with Add
 321             updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 322             bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 323             processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 324         } else {
 325             // Interface is retained in the DPN, but its Link Up.
 326             // Advertise prefixes again for this interface to BGP
 327             advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
 328                     vpnInterface);
 329         }
 330     }
 331
 332
 333 //    private class UpdateDpnToVpnWorker implements Callable<List<ListenableFuture<Void>>> {
 334 //        BigInteger dpnId;
 335 //        String vpnName;
 336 //        String interfaceName;
 337 //        boolean addToDpn;
 338 //        int lPortTag;
 339 //
 340 //        public UpdateDpnToVpnWorker(BigInteger dpnId, String vpnName, String interfaceName,
 341 //                                    int lPortTag, boolean addToDpn) {
 342 //            this.dpnId= dpnId;
 343 //            this.vpnName = vpnName;
 344 //            this.interfaceName = interfaceName;
 345 //            this.lPortTag = lPortTag;
 346 //            this.addToDpn = addToDpn;
 347 //        }
 348 //
 349 //        @Override
 350 //        public List<ListenableFuture<Void>> call() throws Exception {
 351 //            // If another renderer(for eg : CSS) needs to be supported, check can be performed here
 352 //            // to call the respective helpers.
 353 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 354 //            updateDpnDbs(dpnId, vpnName, interfaceName, addToDpn, writeTxn);
 355 //            List<ListenableFuture<Void>> futures = new ArrayList<>();
 356 //            futures.add(writeTxn.submit());
 357 //            ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
 358 //            Futures.addCallback(listenableFuture,
 359 //                    new UpdateDpnToVpnCallback(dpnId, vpnName, interfaceName, lPortTag, addToDpn));
 360 //            return futures;
 361 //        }
 362 //    }
 363 //
 364 //
 365 //    /**
 366 //     * JobCallback class is used as a future callback for
 367 //     * main and rollback workers to handle success and failure.
 368 //     */
 369 //    private class UpdateDpnToVpnCallback implements FutureCallback<List<Void>> {
 370 //        BigInteger dpnId;
 371 //        String vpnName;
 372 //        String interfaceName;
 373 //        boolean addToDpn;
 374 //        int lPortTag;
 375 //
 376 //        public UpdateDpnToVpnCallback(BigInteger dpnId, String vpnName, String interfaceName,
 377 //                                      int lPortTag, boolean addToDpn) {
 378 //            this.dpnId= dpnId;
 379 //            this.vpnName = vpnName;
 380 //            this.interfaceName = interfaceName;
 381 //            this.lPortTag = lPortTag;
 382 //            this.addToDpn = addToDpn;
 383 //        }
 384 //
 385 //        /**
 386 //         * @param voids
 387 //         * This implies that all the future instances have returned success. -- TODO: Confirm this
 388 //         */
 389 //        @Override
 390 //        public void onSuccess(List<Void> voids) {
 391 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 392 //            bindService(dpnId, vpnName, interfaceName, lPortTag, writeTxn);
 393 //            processVpnInterfaceAdjacencies(dpnId, vpnName, interfaceName, writeTxn);
 394 //            writeTxn.submit();
 395 //        }
 396 //
 397 //        /**
 398 //         *
 399 //         * @param throwable
 400 //         * This method is used to handle failure callbacks.
 401 //         * If more retry needed, the retrycount is decremented and mainworker is executed again.
 402 //         * After retries completed, rollbackworker is executed.
 403 //         * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 404 //         */
 405 //
 406 //        @Override
 407 //        public void onFailure(Throwable throwable) {
 408 //            LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 409 //        }
 410 //    }
 411
 412
 413
 414
 415     private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
 416                                                  VpnInterface intf) {
 417         //Read NextHops
 418         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 419         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 420
 421         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 422         if (rd == null) {
 423             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 424                     intf.getName(), intf.getVpnInstanceName());
 425             return;
 426         } else {
 427             if (rd.equals(intf.getVpnInstanceName())) {
 428                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in " +
 429                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 430
 431                 return;
 432             }
 433         }
 434         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
 435                 intf.getVpnInstanceName(), rd);
 436
 437         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 438         if (nextHopIp == null){
 439             LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
 440             return;
 441         }
 442
 443         if (adjacencies.isPresent()) {
 444             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 445
 446             if (!nextHops.isEmpty()) {
 447                 LOG.trace("NextHops are " + nextHops);
 448                 for (Adjacency nextHop : nextHops) {
 449                     long label = nextHop.getLabel();
 450                     try {
 451                         LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 452                         bgpManager.advertisePrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
 453                         LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 454                     } catch(Exception e) {
 455                         LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
 456                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 457                     }
 458                 }
 459             }
 460         }
 461     }
 462
 463     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 464         //Read NextHops
 465         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 466         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 467
 468         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 469         if (rd == null) {
 470             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 471                     intf.getName(), intf.getVpnInstanceName());
 472             return;
 473         } else {
 474             if (rd.equals(intf.getVpnInstanceName())) {
 475                 LOG.info("withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in " +
 476                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 477                 return;
 478             }
 479         }
 480         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 481                 intf.getVpnInstanceName(), rd);
 482         if (adjacencies.isPresent()) {
 483             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 484
 485             if (!nextHops.isEmpty()) {
 486                 LOG.trace("NextHops are " + nextHops);
 487                 for (Adjacency nextHop : nextHops) {
 488                     try {
 489                         LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 490                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 491                         LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 492                     } catch(Exception e) {
 493                         LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
 494                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 495                     }
 496                 }
 497             }
 498         }
 499     }
 500
 501     public void updateVpnToDpnMapping(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 502         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 503         if (dpId == null) {
 504             dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, interfaceName);
 505         }
 506         if(!dpId.equals(BigInteger.ZERO)) {
 507             if(add)
 508                 createOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 509             else
 510                 removeOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 511         }
 512     }
 513
 514     private void bindService(BigInteger dpId, final String vpnInstanceName, final String vpnInterfaceName,
 515                              int lPortTag, WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
 516         final int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 517         final long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
 518
 519         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 520         dataStoreCoordinator.enqueueJob(vpnInterfaceName,
 521                 new Callable<List<ListenableFuture<Void>>>() {
 522                     @Override
 523                     public List<ListenableFuture<Void>> call() throws Exception {
 524                         WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 525                         int instructionKey = 0;
 526                         List<Instruction> instructions = new ArrayList<Instruction>();
 527
 528                         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(
 529                                 MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 530                         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_GW_MAC_TABLE, ++instructionKey));
 531
 532                         BoundServices
 533                                 serviceInfo =
 534                                 InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 535                                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX), priority,
 536                                         NwConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 537                         writeTxn.put(LogicalDatastoreType.CONFIGURATION,
 538                                 InterfaceUtils.buildServiceId(vpnInterfaceName, ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)), serviceInfo, true);
 539                         List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
 540                         futures.add(writeTxn.submit());
 541                         return futures;
 542                     }
 543                 });
 544         makeArpFlow(dpId, ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX), lPortTag, vpnInterfaceName,
 545                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW, writeInvTxn);
 546         setupGwMacIfExternalVpn(dpId, vpnInterfaceName, vpnId, writeInvTxn, NwConstants.ADD_FLOW);
 547         makeArpFlow(dpId, ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX), lPortTag, vpnInterfaceName,
 548                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.ADD_FLOW, writeInvTxn);
 549
 550     }
 551
 552     private void setupGwMacIfExternalVpn(BigInteger dpnId, String interfaceName, long vpnId,
 553             WriteTransaction writeInvTxn, int addOrRemove) {
 554         InstanceIdentifier<VpnIds> vpnIdsInstanceIdentifier = VpnUtil.getVpnIdToVpnInstanceIdentifier(vpnId);
 555         Optional<VpnIds> vpnIdsOptional = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vpnIdsInstanceIdentifier);
 556         if (vpnIdsOptional.isPresent() && vpnIdsOptional.get().isExternalVpn()) {
 557             Optional<String> gwMacAddressOptional = InterfaceUtils.getMacAddressForInterface(dataBroker, interfaceName);
 558             if (!gwMacAddressOptional.isPresent()) {
 559                 LOG.error("Failed to get gwMacAddress for interface {}", interfaceName);
 560                 return;
 561             }
 562             String gwMacAddress = gwMacAddressOptional.get();
 563             FlowEntity flowEntity = VpnUtil.buildL3vpnGatewayFlow(dpnId, gwMacAddress, vpnId);
 564             if (addOrRemove == NwConstants.ADD_FLOW) {
 565                 mdsalManager.addFlowToTx(flowEntity, writeInvTxn);
 566             } else if (addOrRemove == NwConstants.DEL_FLOW) {
 567                 mdsalManager.removeFlowToTx(flowEntity, writeInvTxn);
 568             }
 569         }
 570     }
 571
 572     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, String vpnName, String interfaceName,
 573                                                 WriteTransaction writeConfigTxn,
 574                                                 WriteTransaction writeOperTxn) {
 575         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 576         // Read NextHops
 577         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 578         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 579
 580         if (adjacencies.isPresent()) {
 581             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 582             List<Adjacency> value = new ArrayList<>();
 583
 584             // Get the rd of the vpn instance
 585             String rd = getRouteDistinguisher(vpnName);
 586
 587             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 588             if (nextHopIp == null){
 589                 LOG.error("NextHop for interface {} is null", interfaceName);
 590                 return;
 591             }
 592
 593             List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
 594
 595             LOG.trace("NextHops for interface {} are {}", interfaceName, nextHops);
 596             for (Adjacency nextHop : nextHops) {
 597                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 598                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 599                         VpnUtil.getNextHopLabelKey((rd == null) ? vpnName
 600                                 : rd, prefix));
 601                 List<String> adjNextHop = nextHop.getNextHopIpList();
 602                 value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIpList(
 603                         (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : Arrays.asList(nextHopIp))
 604                         .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 605
 606                 if (nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 607                     LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, interfaceName, vpnName);
 608                     writeOperTxn.merge(
 609                             LogicalDatastoreType.OPERATIONAL,
 610                             VpnUtil.getPrefixToInterfaceIdentifier(
 611                                     VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 612                             VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix), true);
 613                 } else {
 614                     //Extra route adjacency
 615                     LOG.trace("Adding prefix {} and nexthopList {} as extra-route for vpn", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName);
 616                     writeOperTxn.merge(
 617                             LogicalDatastoreType.OPERATIONAL,
 618                             VpnUtil.getVpnToExtrarouteIdentifier(
 619                                     (rd != null) ? rd : vpnName, nextHop.getIpAddress()),
 620                             VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()), true);
 621                 }
 622             }
 623
 624             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 625
 626             VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 627             InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 628             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, true);
 629             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 630
 631             for (Adjacency nextHop : aug.getAdjacency()) {
 632                 long label = nextHop.getLabel();
 633                 if (rd != null) {
 634                     addToLabelMapper(label, dpnId, nextHop.getIpAddress(), Arrays.asList(nextHopIp), vpnId,
 635                             interfaceName, null,false, rd, writeOperTxn);
 636                     addPrefixToBGP(rd, nextHop.getIpAddress(), nextHopIp, label, writeConfigTxn);
 637                     //TODO: ERT - check for VPNs importing my route
 638                     for (VpnInstance vpn : vpnsToImportRoute) {
 639                         String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
 640                         if (vpnRd != null) {
 641                             LOG.debug("Exporting route with rd {} prefix {} nexthop {} label {} to VPN {}", vpnRd, nextHop.getIpAddress(), nextHopIp, label, vpn);
 642                             fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), Arrays.asList(nextHopIp), (int) label,
 643                                     RouteOrigin.SELF_IMPORTED, writeConfigTxn);
 644                         }
 645                     }
 646                 } else {
 647                     // ### add FIB route directly
 648                     fibManager.addOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), Arrays.asList(nextHopIp),
 649                             (int) label, RouteOrigin.STATIC, writeConfigTxn);
 650                 }
 651             }
 652         }
 653     }
 654
 655     private List<VpnInstance> getVpnsImportingMyRoute(final String vpnName) {
 656         List<VpnInstance> vpnsToImportRoute = new ArrayList<>();
 657
 658         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 659                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 660         Optional<VpnInstance> optVpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 661         final VpnInstance vpnInstance;
 662         if (optVpnInstance.isPresent()) {
 663             vpnInstance = optVpnInstance.get();
 664         } else {
 665             LOG.debug("Could not retrieve vpn instance {} to check for vpns importing the routes", vpnName);
 666             return vpnsToImportRoute;
 667         }
 668
 669         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 670             @Override
 671             public boolean apply(VpnInstance input) {
 672                 return !input.getVpnInstanceName().equals(vpnName);
 673             }
 674         };
 675
 676         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 677             @Override
 678             public boolean apply(VpnInstance input) {
 679                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ExportExtcommunity),
 680                         getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 681                 return Iterators.size(commonRTs.iterator()) > 0;
 682             }
 683         };
 684
 685         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 686             @Override
 687             public String apply(VpnInstance vpnInstance) {
 688                 //return vpnInstance.getVpnInstanceName();
 689                 return vpnInstance.getIpv4Family().getRouteDistinguisher();
 690             }
 691         };
 692
 693         vpnsToImportRoute = FluentIterable.from(VpnUtil.getAllVpnInstances(dataBroker)).
 694                 filter(excludeVpn).
 695                 filter(matchRTs).toList();
 696         return vpnsToImportRoute;
 697     }
 698
 699     private List<VpnInstance> getVpnsExportingMyRoute(final String vpnName) {
 700         List<VpnInstance> vpnsToExportRoute = new ArrayList<>();
 701
 702         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 703                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 704         Optional<VpnInstance> optVpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 705         final VpnInstance vpnInstance;
 706         if (optVpnInstance.isPresent()) {
 707             vpnInstance = optVpnInstance.get();
 708         } else {
 709             LOG.debug("Could not retrieve vpn instance {} to check for vpns exporting the routes", vpnName);
 710             return vpnsToExportRoute;
 711         }
 712
 713         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 714             @Override
 715             public boolean apply(VpnInstance input) {
 716                 return !input.getVpnInstanceName().equals(vpnName);
 717             }
 718         };
 719
 720         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 721             @Override
 722             public boolean apply(VpnInstance input) {
 723                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ImportExtcommunity),
 724                         getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
 725                 return Iterators.size(commonRTs.iterator()) > 0;
 726             }
 727         };
 728
 729         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 730             @Override
 731             public String apply(VpnInstance vpnInstance) {
 732                 return vpnInstance.getVpnInstanceName();
 733             }
 734         };
 735
 736         vpnsToExportRoute = FluentIterable.from(VpnUtil.getAllVpnInstances(dataBroker)).
 737                 filter(excludeVpn).
 738                 filter(matchRTs).toList();
 739         return vpnsToExportRoute;
 740     }
 741
 742     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
 743         final Predicate<T> inPredicate = Predicates.<T>in(collection2);
 744         return new Iterable<T>() {
 745             @Override
 746             public Iterator<T> iterator() {
 747                 return Iterators.filter(collection1.iterator(), inPredicate);
 748             }
 749         };
 750     }
 751
 752     private List<String> getRts(VpnInstance vpnInstance, VpnTarget.VrfRTType rtType) {
 753         String name = vpnInstance.getVpnInstanceName();
 754         List<String> rts = new ArrayList<>();
 755         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 756         if (vpnConfig == null) {
 757             LOG.trace("vpn config is not available for {}", name);
 758             return rts;
 759         }
 760         VpnTargets targets = vpnConfig.getVpnTargets();
 761         if (targets == null) {
 762             LOG.trace("vpn targets not available for {}", name);
 763             return rts;
 764         }
 765         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 766         if (vpnTargets == null) {
 767             LOG.trace("vpnTarget values not available for {}", name);
 768             return rts;
 769         }
 770         for (VpnTarget target : vpnTargets) {
 771             //TODO: Check for RT type is Both
 772             if(target.getVrfRTType().equals(rtType) ||
 773                     target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
 774                 String rtValue = target.getVrfRTValue();
 775                 rts.add(rtValue);
 776             }
 777         }
 778         return rts;
 779     }
 780
 781     private List<String> getExportRts(VpnInstance vpnInstance) {
 782         List<String> exportRts = new ArrayList<>();
 783         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 784         VpnTargets targets = vpnConfig.getVpnTargets();
 785         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 786         for (VpnTarget target : vpnTargets) {
 787             if (target.getVrfRTType().equals(VpnTarget.VrfRTType.ExportExtcommunity)) {
 788                 String rtValue = target.getVrfRTValue();
 789                 exportRts.add(rtValue);
 790             }
 791         }
 792         return exportRts;
 793     }
 794
 795     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 796                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow,
 797                              WriteTransaction writeConfigTxn){
 798         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 799         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, MetaDataUtil.getVpnIdMetadata(vpnId));
 800         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 801                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 802
 803         // Matching Arp reply flows
 804         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 805         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 806                 metadata, metadataMask }));
 807
 808         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 809
 810         // Instruction to punt to controller
 811         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 812         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 813         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 814         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
 815                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
 816
 817         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 818
 819         // Install the flow entry in L3_INTERFACE_TABLE
 820         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 821                 NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 822         FlowEntity flowEntity;
 823         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 824                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 825                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 826
 827         Flow flow = flowEntity.getFlowBuilder().build();
 828         String flowId = flowEntity.getFlowId();
 829         FlowKey flowKey = new FlowKey( new FlowId(flowId));
 830         Node nodeDpn = VpnUtil.buildDpnNode(dpId);
 831
 832         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 833                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 834                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 835
 836         if (writeConfigTxn != null) {
 837             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 838                 LOG.debug("Creating ARP Flow for interface {}", vpnInterfaceName);
 839                 writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
 840             } else {
 841                 LOG.debug("Deleting ARP Flow for interface {}", vpnInterfaceName);
 842                 writeConfigTxn.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 843             }
 844         } else {
 845             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 846                 LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 847                 mdsalManager.installFlow(flowEntity);
 848             } else {
 849                 LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 850                 mdsalManager.removeFlow(flowEntity);
 851             }
 852         }
 853     }
 854
 855     private String getRouteDistinguisher(String vpnName) {
 856         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 857                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 858         Optional<VpnInstance> vpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 859         String rd = null;
 860         if(vpnInstance.isPresent()) {
 861             VpnInstance instance = vpnInstance.get();
 862             VpnAfConfig config = instance.getIpv4Family();
 863             rd = config.getRouteDistinguisher();
 864         }
 865         return rd;
 866     }
 867
 868     /**
 869      * JobCallback class is used as a future callback for
 870      * main and rollback workers to handle success and failure.
 871      */
 872     private class DpnEnterExitVpnWorker implements FutureCallback<List<Void>> {
 873         BigInteger dpnId;
 874         String vpnName;
 875         String rd;
 876         boolean entered;
 877
 878         public DpnEnterExitVpnWorker(BigInteger dpnId, String vpnName, String rd, boolean entered) {
 879             this.entered = entered;
 880             this.dpnId = dpnId;
 881             this.vpnName = vpnName;
 882             this.rd = rd;
 883         }
 884
 885         /**
 886          * @param voids
 887          * This implies that all the future instances have returned success. -- TODO: Confirm this
 888          */
 889         @Override
 890         public void onSuccess(List<Void> voids) {
 891             if (entered) {
 892                 publishAddNotification(dpnId, vpnName, rd);
 893             } else {
 894                 publishRemoveNotification(dpnId, vpnName, rd);
 895             }
 896         }
 897
 898         /**
 899          *
 900          * @param throwable
 901          * This method is used to handle failure callbacks.
 902          * If more retry needed, the retrycount is decremented and mainworker is executed again.
 903          * After retries completed, rollbackworker is executed.
 904          * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 905          */
 906         @Override
 907         public void onFailure(Throwable throwable) {
 908             LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 909         }
 910     }
 911
 912     private void createOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 913         String routeDistinguisher = getRouteDistinguisher(vpnName);
 914         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 915         Boolean newDpnOnVpn = Boolean.FALSE;
 916
 917         synchronized (vpnName.intern()) {
 918             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 919             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 920             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 921             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data
 922                     .entry.vpn.to.dpn.list.VpnInterfaces
 923                     vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 924
 925             if (dpnInVpn.isPresent()) {
 926                 VpnToDpnList vpnToDpnList = dpnInVpn.get();
 927                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 928                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = vpnToDpnList.getVpnInterfaces();
 929                 if (vpnInterfaces == null) {
 930                     vpnInterfaces = new ArrayList<>();
 931                 }
 932                 vpnInterfaces.add(vpnInterface);
 933                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder(vpnToDpnList);
 934                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 935
 936                 if (writeTxn != null) {
 937                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 938                 } else {
 939                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 940                 }
 941                 /* If earlier state was inactive, it is considered new DPN coming back to the
 942                  * same VPN
 943                  */
 944                 if (vpnToDpnList.getDpnState() == VpnToDpnList.DpnState.Inactive) {
 945                     newDpnOnVpn = Boolean.TRUE;
 946                 }
 947             } else {
 948                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 949                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = new ArrayList<>();
 950                 vpnInterfaces.add(vpnInterface);
 951                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder().setDpnId(dpnId);
 952                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 953
 954                 if (writeTxn != null) {
 955                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 956                 } else {
 957                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 958                 }
 959                 newDpnOnVpn = Boolean.TRUE;
 960             }
 961             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
 962             try {
 963                 futures.get();
 964             } catch (InterruptedException | ExecutionException e) {
 965                 LOG.error("Error adding to dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
 966                 throw new RuntimeException(e.getMessage());
 967             }
 968         }
 969         /*
 970          * Informing the Fib only after writeTxn is submitted successfuly.
 971          */
 972         if (newDpnOnVpn) {
 973             LOG.debug("Sending populateFib event for new dpn {} in VPN {}", dpnId, vpnName);
 974             fibManager.populateFibOnNewDpn(dpnId, vpnId, rd, new DpnEnterExitVpnWorker(dpnId, vpnName, rd, true /* entered */));
 975         }
 976     }
 977
 978     private void removeOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 979         Boolean lastDpnOnVpn = Boolean.FALSE;
 980         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
 981         synchronized (vpnName.intern()) {
 982             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 983             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 984             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 985             if (dpnInVpn.isPresent()) {
 986                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 987                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 988                 org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 989                         currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 990
 991                 if (vpnInterfaces.remove(currVpnInterface)) {
 992                     if (vpnInterfaces.isEmpty()) {
 993                         List<IpAddresses> ipAddresses = dpnInVpn.get().getIpAddresses();
 994                         if (ipAddresses == null || ipAddresses.isEmpty()) {
 995                             VpnToDpnListBuilder dpnInVpnBuilder =
 996                                     new VpnToDpnListBuilder(dpnInVpn.get())
 997                                             .setDpnState(VpnToDpnList.DpnState.Inactive)
 998                                             .setVpnInterfaces(null);
 999                             if (writeTxn != null) {
1000                                 writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build(), true);
1001                             } else {
1002                                 VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build());
1003                             }
1004                             lastDpnOnVpn = Boolean.TRUE;
1005                         } else {
1006                             LOG.warn("vpn interfaces are empty but ip addresses are present for the vpn {} in dpn {}", vpnName, dpnId);
1007                         }
1008                     } else {
1009                         if (writeTxn != null) {
1010                             writeTxn.delete(LogicalDatastoreType.OPERATIONAL, id.child(
1011                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1012                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1013                                     new VpnInterfacesKey(intfName)));
1014                         } else {
1015                             VpnUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1016                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1017                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1018                                     new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
1019                         }
1020                     }
1021                 }
1022             }
1023             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
1024             try {
1025                 futures.get();
1026             } catch (InterruptedException | ExecutionException e) {
1027                 LOG.error("Error removing from dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
1028                 throw new RuntimeException(e.getMessage());
1029             }
1030         }
1031         if (lastDpnOnVpn) {
1032             LOG.debug("Sending cleanup event for dpn {} in VPN {}", dpnId, vpnName);
1033             fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd, new DpnEnterExitVpnWorker(dpnId, vpnName, rd, false /* exited */));
1034         }
1035     }
1036
1037     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1038         List<VpnInstance> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1039         for (VpnInstance vpn : vpnsToExportRoute) {
1040             String rd = vpn.getIpv4Family().getRouteDistinguisher();
1041             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getIpv4Family().getRouteDistinguisher());
1042             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1043             if (vrfEntries != null) {
1044                 for (VrfEntry vrfEntry : vrfEntries) {
1045                     try {
1046                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1047                             continue;
1048                         }
1049                         String prefix = vrfEntry.getDestPrefix();
1050                         long label = vrfEntry.getLabel();
1051                         List<String> nextHops = vrfEntry.getNextHopAddressList();
1052                         SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1053                         for (String nh : nextHops) {
1054                             if (route != null) {
1055                                 LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1056                                 importSubnetRouteForNewVpn(rd, prefix, nh, (int)label, route, writeConfigTxn);
1057                             } else {
1058                                 LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1059                                 fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, prefix, Arrays.asList(nh), (int)label,
1060                                         RouteOrigin.SELF_IMPORTED, writeConfigTxn);
1061                             }
1062                         }
1063                     } catch (Exception e) {
1064                         LOG.error("Exception occurred while importing route with prefix {} label {} nexthop {} from vpn {} to vpn {}", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpn.getVpnInstanceName(), vpnName);
1065                     }
1066                 }
1067                 writeConfigTxn.submit();
1068             } else {
1069                 LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getIpv4Family().getRouteDistinguisher());
1070             }
1071         }
1072     }
1073
1074     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label, WriteTransaction writeConfigTxn) {
1075         try {
1076             LOG.info("ADD: Adding Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1077             fibManager.addOrUpdateFibEntry(dataBroker, rd, prefix, Arrays.asList(nextHopIp), (int)label, RouteOrigin.STATIC, writeConfigTxn);
1078             bgpManager.advertisePrefix(rd, prefix, Arrays.asList(nextHopIp), (int)label);
1079             LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1080         } catch(Exception e) {
1081             LOG.error("Add prefix failed", e);
1082         }
1083     }
1084
1085     @Override
1086     public void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1087         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
1088         LOG.info("VPN Interface remove event - intfName {}" ,vpnInterface.getName());
1089         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1090         final String interfaceName = key.getName();
1091
1092         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1093         final Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1094         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
1095                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1096         if (optVpnInterface.isPresent()){
1097             BigInteger dpnId = BigInteger.ZERO;
1098             Boolean dpnIdRetrieved = Boolean.FALSE;
1099             if(interfaceState != null){
1100                 try{
1101                     dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1102                     dpnIdRetrieved = Boolean.TRUE;
1103                 }catch (Exception e){
1104                     LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", interfaceName, e);
1105                 }
1106             } else {
1107                 LOG.error("Unable to retrieve interfaceState for interface {} , quitting ", interfaceName);
1108                 return;
1109             }
1110             final VpnInterface vpnOpInterface = optVpnInterface.get();
1111             if(dpnIdRetrieved == Boolean.FALSE){
1112                 LOG.info("dpnId for {} has not been retrieved yet. Fetching from vpn interface operational DS", interfaceName);
1113                 dpnId = vpnOpInterface.getDpnId();
1114             }
1115             final int ifIndex = interfaceState.getIfIndex();
1116             final BigInteger dpId = dpnId;
1117             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1118             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1119                     new Callable<List<ListenableFuture<Void>>>() {
1120                         @Override
1121                         public List<ListenableFuture<Void>> call() throws Exception {
1122                             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1123                             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1124                             WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
1125                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, false, true, writeConfigTxn, writeOperTxn, writeInvTxn);
1126                             List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
1127                             futures.add(writeOperTxn.submit());
1128                             futures.add(writeConfigTxn.submit());
1129                             futures.add(writeInvTxn.submit());
1130                             return futures;
1131                         }
1132                     });
1133
1134         }else{
1135             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
1136         }
1137     }
1138
1139     protected void processVpnInterfaceDown(BigInteger dpId,
1140                                            String interfaceName,
1141                                            int lPortTag,
1142                                            boolean isInterfaceStateDown,
1143                                            boolean isConfigRemoval,
1144                                            WriteTransaction writeConfigTxn,
1145                                            WriteTransaction writeOperTxn,
1146                                            WriteTransaction writeInvTxn) {
1147         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1148         if (!isInterfaceStateDown) {
1149             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1150             if(vpnInterface == null){
1151                 LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
1152                 return;
1153             }else{
1154                 final String vpnName = vpnInterface.getVpnInstanceName();
1155                 if(!vpnInterface.isScheduledForRemove()){
1156                     VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE, writeOperTxn);
1157                     removeAdjacenciesFromVpn(dpId, interfaceName, vpnInterface.getVpnInstanceName(), writeConfigTxn);
1158                     LOG.info("Unbinding vpn service from interface {} ", interfaceName);
1159                     unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown, isConfigRemoval, writeConfigTxn, writeInvTxn);
1160                 }else{
1161                     LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ", interfaceName);
1162                     return;
1163                 }
1164             }
1165         } else {
1166             // Interface is retained in the DPN, but its Link Down.
1167             // Only withdraw the prefixes for this interface from BGP
1168             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1169             if(vpnInterface == null){
1170                 LOG.info("Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in operational data store", interfaceName);
1171                 return;
1172             }else {
1173                 withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
1174             }
1175         }
1176     }
1177
1178     private void waitForFibToRemoveVpnPrefix(String interfaceName) {
1179         // FIB didn't get a chance yet to clean up this VPNInterface
1180         // Let us give it a chance here !
1181         LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
1182         try {
1183             Runnable notifyTask = new VpnNotifyTask();
1184             vpnIntfMap.put(interfaceName, notifyTask);
1185             synchronized (notifyTask) {
1186                 try {
1187                     notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
1188                 } catch (InterruptedException e) {
1189                 }
1190             }
1191         } finally {
1192             vpnIntfMap.remove(interfaceName);
1193         }
1194     }
1195
1196     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final String interfaceName, final String vpnName,
1197                                           WriteTransaction writeConfigTxn) {
1198         //Read NextHops
1199         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1200         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1201         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1202
1203         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1204         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", interfaceName,
1205                 vpnName, rd);
1206         if (adjacencies.isPresent()) {
1207             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1208
1209             if (!nextHops.isEmpty()) {
1210                 LOG.trace("NextHops are " + nextHops);
1211                 for (Adjacency nextHop : nextHops) {
1212                     List<String> nhList = new ArrayList<String>();
1213                     if (nextHop.getMacAddress() == null || nextHop.getMacAddress().isEmpty()) {
1214                         // This is either an extra-route (or) a learned IP via subnet-route
1215                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1216                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1217                             LOG.error("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
1218                                     rd, nextHop.getIpAddress());
1219                             continue;
1220                         }
1221                         nhList = Arrays.asList(nextHopIp);
1222                     } else {
1223                         // This is a primary adjacency
1224                         nhList = nextHop.getNextHopIpList();
1225                     }
1226                     if (rd.equals(vpnName)) {
1227                         //this is an internal vpn - the rd is assigned to the vpn instance name;
1228                         //remove from FIB directly
1229                         for(String nh : nhList) {
1230                             fibManager.removeOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), nh, writeConfigTxn);
1231                         }
1232                     } else {
1233                         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1234                         for (String nh : nhList) {
1235                             //IRT: remove routes from other vpns importing it
1236                             removePrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1237                             for (VpnInstance vpn : vpnsToImportRoute) {
1238                                 String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
1239                                 if (vpnRd != null) {
1240                                     LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd, nextHop.getIpAddress(), vpn.getVpnInstanceName());
1241                                     fibManager.removeOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), nh, writeConfigTxn);
1242                                 }
1243                             }
1244                         }
1245                     }
1246                     String ip = nextHop.getIpAddress().split("/")[0];
1247                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
1248                             vpnName, ip);
1249                     if (vpnPortipToPort != null && !vpnPortipToPort.isConfig()) {
1250                         LOG.trace("VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
1251                                 vpnPortipToPort.getPortName(),ip);
1252                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip);
1253                     }
1254                 }
1255             }
1256         }
1257     }
1258
1259
1260     private void unbindService(BigInteger dpId, String vpnInstanceName, final String vpnInterfaceName,
1261                                int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval,
1262                                WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
1263         short l3vpn_service_index = ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX);
1264         if (!isInterfaceStateDown && isConfigRemoval) {
1265             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1266             dataStoreCoordinator.enqueueJob(vpnInterfaceName,
1267                     new Callable<List<ListenableFuture<Void>>>() {
1268                         @Override
1269                         public List<ListenableFuture<Void>> call() throws Exception {
1270                             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
1271                             writeTxn.delete(LogicalDatastoreType.CONFIGURATION,
1272                                     InterfaceUtils.buildServiceId(vpnInterfaceName,
1273                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)));
1274
1275                             List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
1276                             futures.add(writeTxn.submit());
1277                             return futures;
1278                         }
1279                     });
1280         }
1281         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
1282         setupGwMacIfExternalVpn(dpId, vpnInterfaceName, vpnId, writeConfigTxn, NwConstants.DEL_FLOW);
1283         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
1284                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW, writeInvTxn);
1285         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
1286                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.DEL_FLOW, writeInvTxn);
1287     }
1288
1289
1290     private void removePrefixFromBGP(String rd, String prefix, String nextHop, WriteTransaction writeConfigTxn) {
1291         try {
1292             LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
1293             fibManager.removeOrUpdateFibEntry(dataBroker, rd, prefix, nextHop, writeConfigTxn);
1294             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1295             LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
1296         } catch(Exception e) {
1297             LOG.error("Delete prefix failed", e);
1298         }
1299     }
1300
1301     @Override
1302     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
1303         LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
1304         LOG.info("VPN Interface update event - intfName {}" ,update.getName());
1305         String oldVpnName = original.getVpnInstanceName();
1306         String newVpnName = update.getVpnInstanceName();
1307         BigInteger dpnId = update.getDpnId();
1308         List<Adjacency> oldAdjsList = new ArrayList<>();
1309         List<Adjacency> newAdjsList = new ArrayList<>();
1310         Adjacencies oldAdjs = original.getAugmentation(Adjacencies.class);
1311         Adjacencies newAdjs = update.getAugmentation(Adjacencies.class);
1312         if (oldAdjs != null) {
1313             oldAdjsList = oldAdjs.getAdjacency();
1314         }
1315         if (newAdjs != null) {
1316             newAdjsList = newAdjs.getAdjacency();
1317         }
1318         //handles switching between <internal VPN - external VPN>
1319         if (!oldVpnName.equals(newVpnName)) {
1320             remove(identifier, original);
1321             waitForFibToRemoveVpnPrefix(update.getName());
1322             add(identifier, update);
1323         }
1324         //handle both addition and removal of adjacencies
1325         //currently, new adjacency may be an extra route
1326         if (!oldAdjsList.equals(newAdjsList)) {
1327             for (Adjacency adj : newAdjsList) {
1328                 if (oldAdjsList.contains(adj)) {
1329                     oldAdjsList.remove(adj);
1330                 } else {
1331                     // add new adjacency - right now only extra route will hit this path
1332                     addNewAdjToVpnInterface(identifier, adj, dpnId);
1333                 }
1334             }
1335             for (Adjacency adj : oldAdjsList) {
1336                 delAdjFromVpnInterface(identifier, adj, dpnId);
1337             }
1338         }
1339     }
1340
1341     public void processArpRequest(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715
1342                                           .IpAddress srcIP, PhysAddress srcMac, org.opendaylight.yang.gen.v1.urn.ietf
1343             .params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress targetIP, PhysAddress targetMac, String srcInterface){
1344         //Build ARP response with ARP requests TargetIp TargetMac as the Arp Response SrcIp and SrcMac
1345         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
1346                 .setDstIpaddress(srcIP).setDstMacaddress(srcMac).setSrcIpaddress(targetIP).setSrcMacaddress(targetMac).build();
1347         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
1348         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
1349         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
1350             @Override
1351             public void onFailure(Throwable error) {
1352                 LOG.error("Error - {}", msgFormat, error);
1353             }
1354
1355             @Override
1356             public void onSuccess(RpcResult<Void> result) {
1357                 if(!result.isSuccessful()) {
1358                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
1359                 } else {
1360                     LOG.debug("Successful RPC Result - {}", msgFormat);
1361                 }
1362             }
1363         });
1364     }
1365
1366     private String getErrorText(Collection<RpcError> errors) {
1367         StringBuilder errorText = new StringBuilder();
1368         for(RpcError error : errors) {
1369             errorText.append(",").append(error.getErrorType()).append("-")
1370                     .append(error.getMessage());
1371         }
1372         return errorText.toString();
1373     }
1374
1375     private void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1376                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd,
1377                                   WriteTransaction writeOperTxn) {
1378         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1379         Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
1380         Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
1381         Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
1382         if (!isSubnetRoute) {
1383             // NextHop must be present for non-subnetroute entries
1384             Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
1385         }
1386         LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
1387         if (dpnId != null) {
1388             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1389                     .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
1390             LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1391             lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(vpnId)
1392                     .setIsSubnetRoute(isSubnetRoute);
1393             if (elanTag != null) {
1394                 lriBuilder.setElanTag(elanTag);
1395             }
1396             if (vpnInterfaceName != null) {
1397                 lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1398             }
1399             lriBuilder.setParentVpnRd(rd);
1400             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
1401             if (vpnInstanceOpDataEntry != null) {
1402                 List<String> vpnInstanceNames = Arrays.asList(vpnInstanceOpDataEntry.getVpnInstanceName());
1403                 lriBuilder.setVpnInstanceList(vpnInstanceNames);
1404             }
1405             LabelRouteInfo lri = lriBuilder.build();
1406             LOG.trace("Adding route info to label map: {}", lri);
1407             if (writeOperTxn != null) {
1408                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, lriIid, lri, true);
1409             } else {
1410                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
1411             }
1412         } else {
1413             LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
1414         }
1415     }
1416
1417     public synchronized void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
1418                                                         long elantag, BigInteger dpnId, WriteTransaction writeTxn) {
1419         SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
1420         RouteOrigin origin = RouteOrigin.STATIC; // Only case when a route is considered as directly connected
1421         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1422                 .setLabel((long)label).setOrigin(origin.getValue())
1423                 .addAugmentation(SubnetRoute.class, route).build();
1424
1425         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
1426
1427         //TODO: What should be parentVpnId? Get it from RD?
1428         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1429         addToLabelMapper((long)label, dpnId, prefix, Arrays.asList(nextHop), vpnId, null, elantag, true, rd, null);
1430         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1431
1432         InstanceIdentifierBuilder<VrfTables> idBuilder =
1433                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1434         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1435
1436         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1437                 setVrfEntry(vrfEntryList).build();
1438
1439         if (writeTxn != null) {
1440             writeTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1441         } else {
1442             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1443         }
1444
1445         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1446         if (vpnsToImportRoute.size() > 0) {
1447             VrfEntry importingVrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1448                     .setLabel((long) label).setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
1449                     .addAugmentation(SubnetRoute.class, route).build();
1450             List<VrfEntry> importingVrfEntryList = Arrays.asList(importingVrfEntry);
1451             for (VpnInstance vpnInstance : vpnsToImportRoute) {
1452                 LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop, label, vpnInstance.getVpnInstanceName());
1453                 String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1454                 InstanceIdentifier<VrfTables> importingVrfTableId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(importingRd)).build();
1455                 VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(importingVrfEntryList).build();
1456                 if (writeTxn != null) {
1457                     writeTxn.merge(LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable, true);
1458                 } else {
1459                     VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable);
1460                 }
1461             }
1462         }
1463     }
1464
1465     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1466                                                         SubnetRoute route, WriteTransaction writeConfigTxn) {
1467
1468         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1469         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1470                 .setLabel((long)label).setOrigin(origin.getValue())
1471                 .addAugmentation(SubnetRoute.class, route).build();
1472         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, route.getElantag());
1473         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1474         InstanceIdentifierBuilder<VrfTables> idBuilder =
1475                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1476         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1477         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1478                 setVrfEntry(vrfEntryList).build();
1479         if (writeConfigTxn != null) {
1480             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1481         } else {
1482             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1483         }
1484     }
1485
1486     public synchronized void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName){
1487         fibManager.removeFibEntry(dataBroker, rd, prefix, null);
1488         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1489         for (VpnInstance vpnInstance : vpnsToImportRoute) {
1490             String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1491             LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix, vpnInstance.getVpnInstanceName());
1492             fibManager.removeFibEntry(dataBroker, importingRd, prefix, null);
1493         }
1494     }
1495
1496     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1497
1498         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1499
1500         if (optVpnInterface.isPresent()) {
1501             VpnInterface currVpnIntf = optVpnInterface.get();
1502             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1503             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1504             rd = (rd != null) ? rd : currVpnIntf.getVpnInstanceName();
1505             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1506             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, adjPath);
1507             long label =
1508                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1509                             VpnUtil.getNextHopLabelKey(rd, prefix));
1510
1511             List<Adjacency> adjacencies;
1512             if (optAdjacencies.isPresent()) {
1513                 adjacencies = optAdjacencies.get().getAdjacency();
1514             } else {
1515                 //This code will not be hit since VM adjacency will always be there
1516                 adjacencies = new ArrayList<>();
1517             }
1518
1519             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIpList(adj.getNextHopIpList())
1520                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
1521
1522             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1523             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove());
1524
1525             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1526             for (String nh : adj.getNextHopIpList()) {
1527                 addExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(), (int) label,
1528                         currVpnIntf.getName());
1529             }
1530         }
1531     }
1532
1533     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1534         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1535
1536         if (optVpnInterface.isPresent()) {
1537             VpnInterface currVpnIntf = optVpnInterface.get();
1538
1539             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1540             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1541             if (optAdjacencies.isPresent()) {
1542                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1543
1544                 if (!adjacencies.isEmpty()) {
1545                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1546                     rd = (rd != null) ? rd :currVpnIntf.getVpnInstanceName();
1547                     LOG.trace("Adjacencies are " + adjacencies);
1548                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1549                     while (adjIt.hasNext()) {
1550                         Adjacency adjElem = adjIt.next();
1551                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1552                             adjIt.remove();
1553
1554                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1555                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1556                                     currVpnIntf.getVpnInstanceName(),
1557                                     aug, dpnId, currVpnIntf.isScheduledForRemove());
1558
1559                             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1560
1561                             for (String nh : adj.getNextHopIpList()) {
1562                                 delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
1563                                         currVpnIntf.getName());
1564                             }
1565                             break;
1566                         }
1567
1568                     }
1569                 }
1570             }
1571         }
1572
1573     }
1574
1575     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label,
1576                                  String intfName) {
1577
1578         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1579         VpnUtil.syncUpdate(
1580                 dataBroker,
1581                 LogicalDatastoreType.OPERATIONAL,
1582                 VpnUtil.getVpnToExtrarouteIdentifier( (rd != null) ? rd : routerID, destination),
1583                 VpnUtil.getVpnToExtraroute(destination, Arrays.asList(nextHop)));
1584
1585         BigInteger dpnId = null;
1586         if (intfName != null && !intfName.isEmpty()) {
1587             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1588             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1589             if (nextHopIp == null || nextHopIp.isEmpty()) {
1590                 LOG.error("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1591                         intfName, destination);
1592                 return;
1593             }
1594             nextHop = nextHopIp;
1595         }
1596         List<String> nextHopIpList = Arrays.asList(nextHop);
1597         if (rd != null) {
1598             /* Label mapper is required only for BGP VPN and not for Internal VPN */
1599             addToLabelMapper((long) label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(dataBroker, routerID),
1600                     intfName, null, false, rd, null);
1601         }
1602
1603         // TODO (eperefr): This is a limitation to be stated in docs. When configuring static route to go to
1604         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1605         // first place.
1606         Optional<InterVpnLink> optInterVpnLink = InterVpnLinkUtil.getInterVpnLinkByEndpointIp(dataBroker, nextHop);
1607         if ( optInterVpnLink.isPresent() ) {
1608             InterVpnLink interVpnLink = optInterVpnLink.get();
1609             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1610             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1611             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1612             // This is like leaking one of the Vpn2 routes towards Vpn1
1613             boolean nexthopIsVpn2 = ( interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nextHop) );
1614             String srcVpnUuid = (nexthopIsVpn2) ? interVpnLink.getSecondEndpoint().getVpnUuid().getValue()
1615                     : interVpnLink.getFirstEndpoint().getVpnUuid().getValue();
1616             String dstVpnUuid = (nexthopIsVpn2) ? interVpnLink.getFirstEndpoint().getVpnUuid().getValue()
1617                     : interVpnLink.getSecondEndpoint().getVpnUuid().getValue();
1618             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
1619             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1620                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1621             InterVpnLinkUtil.leakRoute(dataBroker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel);
1622         } else {
1623             if (rd != null) {
1624                 addPrefixToBGP(rd, destination, nextHop, label, null);
1625             } else {
1626                 // ### add FIB route directly
1627                 fibManager.addOrUpdateFibEntry(dataBroker, routerID, destination, Arrays.asList(nextHop), label, RouteOrigin.STATIC, null);
1628             }
1629         }
1630     }
1631
1632     protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName) {
1633         if (intfName != null && !intfName.isEmpty()) {
1634             BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1635             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1636             if (nextHopIp == null || nextHopIp.isEmpty()) {
1637                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1638                         intfName, destination);
1639             }
1640             nextHop = nextHopIp;
1641         }
1642
1643         if (rd != null) {
1644             removePrefixFromBGP(rd, destination, nextHop, null);
1645         } else {
1646             // ### add FIB route directly
1647             fibManager.removeOrUpdateFibEntry(dataBroker, routerID, destination, nextHop, null);
1648         }
1649     }
1650
1651     void publishAddNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1652         LOG.debug("Sending notification for add dpn {} in vpn {} event ", dpnId, vpnName);
1653         AddEventData data = new AddEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1654         AddDpnEvent event = new AddDpnEventBuilder().setAddEventData(data).build();
1655         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1656         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1657             @Override
1658             public void onFailure(Throwable error) {
1659                 LOG.warn("Error in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName, error);
1660             }
1661
1662             @Override
1663             public void onSuccess(Object arg) {
1664                 LOG.trace("Successful in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName);
1665             }
1666         });
1667     }
1668
1669     void publishRemoveNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1670         LOG.debug("Sending notification for remove dpn {} in vpn {} event ", dpnId, vpnName);
1671         RemoveEventData data = new RemoveEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1672         RemoveDpnEvent event = new RemoveDpnEventBuilder().setRemoveEventData(data).build();
1673         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1674         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1675             @Override
1676             public void onFailure(Throwable error) {
1677                 LOG.warn("Error in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName, error);
1678             }
1679
1680             @Override
1681             public void onSuccess(Object arg) {
1682                 LOG.trace("Successful in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName);
1683             }
1684         });
1685     }
1686
1687     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1688         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1689                 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1690                 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1691     }
1692
1693     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1694         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1695                 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1696     }
1697
1698     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1699         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1700         if(dpId.equals(BigInteger.ZERO)) {
1701             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
1702             return;
1703         }
1704         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1705
1706         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1707                 .OPERATIONAL, routerDpnListIdentifier);
1708         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1709         if (optionalRouterDpnList.isPresent()) {
1710             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1711                     RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1712         } else {
1713             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1714             builder.setRouterId(routerName);
1715             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1716             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
1717             routerInterfaces.add(routerInterface);
1718             builder.setDpnVpninterfacesList(Arrays.asList(dpnVpnList.build()));
1719             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1720                     getRouterId(routerName),
1721                     builder.build(), true);
1722         }
1723     }
1724
1725     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1726         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1727         if(dpId.equals(BigInteger.ZERO)) {
1728             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1729             return;
1730         }
1731         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1732         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1733                 .OPERATIONAL, routerDpnListIdentifier);
1734         if (optionalRouterDpnList.isPresent()) {
1735             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1736             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1737
1738             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1739                 if (routerInterfaces.isEmpty()) {
1740                     if (writeOperTxn != null) {
1741                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1742                     } else {
1743                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1744                     }
1745                 } else {
1746                     if (writeOperTxn != null) {
1747                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1748                                 RouterInterfaces.class,
1749                                 new RouterInterfacesKey(vpnInterfaceName)));
1750                     } else {
1751                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1752                                 RouterInterfaces.class,
1753                                 new RouterInterfacesKey(vpnInterfaceName)));
1754                     }
1755                 }
1756             }
1757         }
1758     }
1759
1760     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1761                                                   WriteTransaction writeOperTxn) {
1762         if(dpId.equals(BigInteger.ZERO)) {
1763             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1764             return;
1765         }
1766         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1767         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1768                 .OPERATIONAL, routerDpnListIdentifier);
1769         if (optionalRouterDpnList.isPresent()) {
1770             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1771             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1772             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1773                 if (routerInterfaces.isEmpty()) {
1774                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1775                 } else {
1776                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1777                             RouterInterfaces.class,
1778                             new RouterInterfacesKey(vpnInterfaceName)));
1779                 }
1780             }
1781         }
1782     }
1783
1784     void notifyTaskIfRequired(String intfName) {
1785         Runnable notifyTask = vpnIntfMap.remove(intfName);
1786         if (notifyTask == null) {
1787             return;
1788         }
1789         executorService.execute(notifyTask);
1790     }
1791 }