vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Function;
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Preconditions;
  13 import com.google.common.base.Predicate;
  14 import com.google.common.base.Predicates;
  15 import com.google.common.collect.FluentIterable;
  16 import com.google.common.collect.Iterators;
  17 import com.google.common.util.concurrent.CheckedFuture;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.JdkFutureAdapters;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import java.math.BigInteger;
  23 import java.util.ArrayList;
  24 import java.util.Arrays;
  25 import java.util.Collection;
  26 import java.util.Iterator;
  27 import java.util.List;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ExecutionException;
  31 import java.util.concurrent.ExecutorService;
  32 import java.util.concurrent.Executors;
  33 import java.util.concurrent.Future;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  36 import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
  37 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  38 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  41 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  42 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  43 import org.opendaylight.genius.mdsalutil.ActionInfo;
  44 import org.opendaylight.genius.mdsalutil.ActionType;
  45 import org.opendaylight.genius.mdsalutil.FlowEntity;
  46 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  47 import org.opendaylight.genius.mdsalutil.InstructionType;
  48 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  49 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  50 import org.opendaylight.genius.mdsalutil.MatchInfo;
  51 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  52 import org.opendaylight.genius.mdsalutil.NwConstants;
  53 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  54 import org.opendaylight.genius.utils.ServiceIndex;
  55 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  56 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  57 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  58 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  59 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.VpnTargets;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInputBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEvent;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEventBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEvent;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEventBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventData;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventDataBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventData;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventDataBuilder;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 131 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 132 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 133 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 134 import org.opendaylight.yangtools.yang.common.RpcError;
 135 import org.opendaylight.yangtools.yang.common.RpcResult;
 136 import org.slf4j.Logger;
 137 import org.slf4j.LoggerFactory;
 138
 139 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
 140     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 141     private ListenerRegistration<DataChangeListener> listenerRegistration;
 142     private final DataBroker dataBroker;
 143     private final IBgpManager bgpManager;
 144     private final IFibManager fibManager;
 145     private final IMdsalApiManager mdsalManager;
 146     private final IdManagerService idManager;
 147     private final OdlArputilService arpManager;
 148     private final OdlInterfaceRpcService ifaceMgrRpcService;
 149     private final NotificationPublishService notificationPublishService;
 150     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
 151     private ExecutorService executorService = Executors.newSingleThreadExecutor();
 152
 153     public VpnInterfaceManager(final DataBroker dataBroker,
 154                                final IBgpManager bgpManager,
 155                                final OdlArputilService arpManager,
 156                                final IdManagerService idManager,
 157                                final IMdsalApiManager mdsalManager,
 158                                final IFibManager fibManager,
 159                                final OdlInterfaceRpcService ifaceMgrRpcService,
 160                                final NotificationPublishService notificationPublishService) {
 161         super(VpnInterface.class);
 162         this.dataBroker = dataBroker;
 163         this.bgpManager = bgpManager;
 164         this.arpManager = arpManager;
 165         this.idManager = idManager;
 166         this.mdsalManager = mdsalManager;
 167         this.fibManager = fibManager;
 168         this.ifaceMgrRpcService = ifaceMgrRpcService;
 169         this.notificationPublishService = notificationPublishService;
 170     }
 171
 172     public void start() {
 173         LOG.info("{} start", getClass().getSimpleName());
 174         listenerRegistration = dataBroker.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 175                 getWildCardPath(), this, DataChangeScope.SUBTREE);
 176     }
 177
 178     private InstanceIdentifier<VpnInterface> getWildCardPath() {
 179         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 180     }
 181
 182     @Override
 183     public void close() throws Exception {
 184         if (listenerRegistration != null) {
 185             listenerRegistration.close();
 186             listenerRegistration = null;
 187         }
 188         LOG.info("{} close", getClass().getSimpleName());
 189     }
 190
 191     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 192         return InstanceIdentifier.create(InterfacesState.class)
 193                 .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 194     }
 195
 196     @Override
 197     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 198         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 199         LOG.info("VPN Interface add event - intfName {}" ,vpnInterface.getName());
 200         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 201         final String interfaceName = key.getName();
 202
 203         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 204                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 205         if(interfaceState != null){
 206             try{
 207                 final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 208                 final int ifIndex = interfaceState.getIfIndex();
 209                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 210                 dataStoreCoordinator.enqueueJob("VPNINTERFACE-"+ interfaceName,
 211                         new Callable<List<ListenableFuture<Void>>>() {
 212                             @Override
 213                             public List<ListenableFuture<Void>> call() throws Exception {
 214                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 215                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 216                                 WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 217                                 processVpnInterfaceUp(dpnId, vpnInterface, ifIndex, false, writeConfigTxn, writeOperTxn, writeInvTxn);
 218                                 List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
 219                                 futures.add(writeOperTxn.submit());
 220                                 futures.add(writeConfigTxn.submit());
 221                                 futures.add(writeInvTxn.submit());
 222                                 return futures;
 223                             }
 224                         });
 225             }catch (Exception e){
 226                 LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", interfaceName, e);
 227                 return;
 228             }
 229         } else {
 230             LOG.info("Handling addition of VPN interface {} skipped as interfaceState is not available", interfaceName);
 231         }
 232     }
 233
 234     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface,
 235                                          final int lPortTag, boolean isInterfaceUp,
 236                                          WriteTransaction writeConfigTxn,
 237                                          WriteTransaction writeOperTxn,
 238                                          WriteTransaction writeInvTxn) {
 239
 240         final String interfaceName = vpnInterface.getName();
 241         if (!isInterfaceUp) {
 242             final String vpnName = vpnInterface.getVpnInstanceName();
 243             LOG.info("Binding vpn service to interface {} ", interfaceName);
 244             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 245             if (vpnId == VpnConstants.INVALID_ID) {
 246                 LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 247                 return;
 248             }
 249             boolean waitForVpnInterfaceOpRemoval = false;
 250             VpnInterface opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, vpnInterface.getName());
 251             if (opVpnInterface != null ) {
 252                 String opVpnName = opVpnInterface.getVpnInstanceName();
 253                 String primaryInterfaceIp = null;
 254                 if(opVpnName.equals(vpnName)) {
 255                     // Please check if the primary VRF Entry does not exist for VPNInterface
 256                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 257                     // back to back
 258                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 259                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 260                     if (adjs == null) {
 261                         LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained", interfaceName);
 262                         return;
 263                     }
 264                     for (Adjacency adj : adjs) {
 265                         if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
 266                             primaryInterfaceIp = adj.getIpAddress();
 267                             break;
 268                         }
 269                     }
 270                     if (primaryInterfaceIp == null) {
 271                         LOG.info("VPN Interface {} addition failed as primary adjacency "
 272                                 + "for this vpn interface could not be obtained", interfaceName);
 273                         return;
 274                     }
 275                     // Get the rd of the vpn instance
 276                     String rd = getRouteDistinguisher(opVpnName);
 277                     rd =  (rd == null) ? opVpnName : rd;
 278                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, rd, primaryInterfaceIp);
 279                     if (vrf != null) {
 280                         LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
 281                         return;
 282                     }
 283                     waitForVpnInterfaceOpRemoval = true;
 284                 } else {
 285                     LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
 286                             interfaceName, vpnName, opVpnName);
 287                 }
 288             }
 289             if (!waitForVpnInterfaceOpRemoval) {
 290                 // Add the VPNInterface and quit
 291                 updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 292                 bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 293                 processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 294                 return;
 295             }
 296
 297             // FIB didn't get a chance yet to clean up this VPNInterface
 298             // Let us give it a chance here !
 299             LOG.info("Trying to add VPN Interface {}, but waiting for FIB to clean up! ", interfaceName);
 300             try {
 301                 Runnable notifyTask = new VpnNotifyTask();
 302                 vpnIntfMap.put(interfaceName, notifyTask);
 303                 synchronized (notifyTask) {
 304                     try {
 305                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 306                     } catch (InterruptedException e) {
 307                     }
 308                 }
 309             } finally {
 310                 vpnIntfMap.remove(interfaceName);
 311             }
 312
 313             opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
 314             if (opVpnInterface != null) {
 315                 LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
 316                 return;
 317             }
 318             // VPNInterface got removed, proceed with Add
 319             updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 320             bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 321             processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 322         } else {
 323             // Interface is retained in the DPN, but its Link Up.
 324             // Advertise prefixes again for this interface to BGP
 325             advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
 326                     vpnInterface);
 327         }
 328     }
 329
 330
 331 //    private class UpdateDpnToVpnWorker implements Callable<List<ListenableFuture<Void>>> {
 332 //        BigInteger dpnId;
 333 //        String vpnName;
 334 //        String interfaceName;
 335 //        boolean addToDpn;
 336 //        int lPortTag;
 337 //
 338 //        public UpdateDpnToVpnWorker(BigInteger dpnId, String vpnName, String interfaceName,
 339 //                                    int lPortTag, boolean addToDpn) {
 340 //            this.dpnId= dpnId;
 341 //            this.vpnName = vpnName;
 342 //            this.interfaceName = interfaceName;
 343 //            this.lPortTag = lPortTag;
 344 //            this.addToDpn = addToDpn;
 345 //        }
 346 //
 347 //        @Override
 348 //        public List<ListenableFuture<Void>> call() throws Exception {
 349 //            // If another renderer(for eg : CSS) needs to be supported, check can be performed here
 350 //            // to call the respective helpers.
 351 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 352 //            updateDpnDbs(dpnId, vpnName, interfaceName, addToDpn, writeTxn);
 353 //            List<ListenableFuture<Void>> futures = new ArrayList<>();
 354 //            futures.add(writeTxn.submit());
 355 //            ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
 356 //            Futures.addCallback(listenableFuture,
 357 //                    new UpdateDpnToVpnCallback(dpnId, vpnName, interfaceName, lPortTag, addToDpn));
 358 //            return futures;
 359 //        }
 360 //    }
 361 //
 362 //
 363 //    /**
 364 //     * JobCallback class is used as a future callback for
 365 //     * main and rollback workers to handle success and failure.
 366 //     */
 367 //    private class UpdateDpnToVpnCallback implements FutureCallback<List<Void>> {
 368 //        BigInteger dpnId;
 369 //        String vpnName;
 370 //        String interfaceName;
 371 //        boolean addToDpn;
 372 //        int lPortTag;
 373 //
 374 //        public UpdateDpnToVpnCallback(BigInteger dpnId, String vpnName, String interfaceName,
 375 //                                      int lPortTag, boolean addToDpn) {
 376 //            this.dpnId= dpnId;
 377 //            this.vpnName = vpnName;
 378 //            this.interfaceName = interfaceName;
 379 //            this.lPortTag = lPortTag;
 380 //            this.addToDpn = addToDpn;
 381 //        }
 382 //
 383 //        /**
 384 //         * @param voids
 385 //         * This implies that all the future instances have returned success. -- TODO: Confirm this
 386 //         */
 387 //        @Override
 388 //        public void onSuccess(List<Void> voids) {
 389 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 390 //            bindService(dpnId, vpnName, interfaceName, lPortTag, writeTxn);
 391 //            processVpnInterfaceAdjacencies(dpnId, vpnName, interfaceName, writeTxn);
 392 //            writeTxn.submit();
 393 //        }
 394 //
 395 //        /**
 396 //         *
 397 //         * @param throwable
 398 //         * This method is used to handle failure callbacks.
 399 //         * If more retry needed, the retrycount is decremented and mainworker is executed again.
 400 //         * After retries completed, rollbackworker is executed.
 401 //         * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 402 //         */
 403 //
 404 //        @Override
 405 //        public void onFailure(Throwable throwable) {
 406 //            LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 407 //        }
 408 //    }
 409
 410
 411
 412
 413     private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
 414                                                  VpnInterface intf) {
 415         //Read NextHops
 416         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 417         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 418
 419         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 420         if (rd == null) {
 421             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 422                     intf.getName(), intf.getVpnInstanceName());
 423             return;
 424         } else {
 425             if (rd.equals(intf.getVpnInstanceName())) {
 426                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in " +
 427                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 428
 429                 return;
 430             }
 431         }
 432         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
 433                 intf.getVpnInstanceName(), rd);
 434
 435         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 436         if (nextHopIp == null){
 437             LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
 438             return;
 439         }
 440
 441         if (adjacencies.isPresent()) {
 442             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 443
 444             if (!nextHops.isEmpty()) {
 445                 LOG.trace("NextHops are " + nextHops);
 446                 for (Adjacency nextHop : nextHops) {
 447                     long label = nextHop.getLabel();
 448                     try {
 449                         LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 450                         bgpManager.advertisePrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
 451                         LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 452                     } catch(Exception e) {
 453                         LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
 454                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 455                     }
 456                 }
 457             }
 458         }
 459     }
 460
 461     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 462         //Read NextHops
 463         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 464         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 465
 466         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 467         if (rd == null) {
 468             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 469                     intf.getName(), intf.getVpnInstanceName());
 470             return;
 471         } else {
 472             if (rd.equals(intf.getVpnInstanceName())) {
 473                 LOG.info("withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in " +
 474                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 475                 return;
 476             }
 477         }
 478         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 479                 intf.getVpnInstanceName(), rd);
 480         if (adjacencies.isPresent()) {
 481             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 482
 483             if (!nextHops.isEmpty()) {
 484                 LOG.trace("NextHops are " + nextHops);
 485                 for (Adjacency nextHop : nextHops) {
 486                     try {
 487                         LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 488                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 489                         LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 490                     } catch(Exception e) {
 491                         LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
 492                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 493                     }
 494                 }
 495             }
 496         }
 497     }
 498
 499     public void updateVpnToDpnMapping(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 500         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 501         if (dpId == null) {
 502             dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, interfaceName);
 503         }
 504         if(!dpId.equals(BigInteger.ZERO)) {
 505             if(add)
 506                 createOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 507             else
 508                 removeOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 509         }
 510     }
 511
 512     private void bindService(BigInteger dpId, final String vpnInstanceName, final String vpnInterfaceName,
 513                              int lPortTag, WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
 514         final int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 515         final long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
 516
 517         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 518         dataStoreCoordinator.enqueueJob(vpnInterfaceName,
 519                 new Callable<List<ListenableFuture<Void>>>() {
 520                     @Override
 521                     public List<ListenableFuture<Void>> call() throws Exception {
 522                         WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 523                         int instructionKey = 0;
 524                         List<Instruction> instructions = new ArrayList<Instruction>();
 525
 526                         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(
 527                                 BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 528                         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 529
 530                         BoundServices
 531                                 serviceInfo =
 532                                 InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 533                                         NwConstants.L3VPN_SERVICE_INDEX, priority,
 534                                         NwConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 535                         writeTxn.put(LogicalDatastoreType.CONFIGURATION,
 536                                 InterfaceUtils.buildServiceId(vpnInterfaceName, NwConstants.L3VPN_SERVICE_INDEX), serviceInfo, true);
 537                         List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
 538                         futures.add(writeTxn.submit());
 539                         return futures;
 540                     }
 541                 });
 542         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
 543                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW, writeInvTxn);
 544         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
 545                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.ADD_FLOW, writeInvTxn);
 546
 547     }
 548
 549     private void processVpnInterfaceAdjacencies(BigInteger dpnId, String vpnName, String interfaceName,
 550                                                 WriteTransaction writeConfigTxn,
 551                                                 WriteTransaction writeOperTxn) {
 552         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 553         // Read NextHops
 554         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 555         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 556
 557         if (adjacencies.isPresent()) {
 558             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 559             List<Adjacency> value = new ArrayList<>();
 560
 561             // Get the rd of the vpn instance
 562             String rd = getRouteDistinguisher(vpnName);
 563
 564             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 565             if (nextHopIp == null){
 566                 LOG.error("NextHop for interface {} is null", interfaceName);
 567                 return;
 568             }
 569
 570             List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
 571
 572             LOG.trace("NextHops for interface {} are {}", interfaceName, nextHops);
 573             for (Adjacency nextHop : nextHops) {
 574                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 575                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 576                         VpnUtil.getNextHopLabelKey((rd == null) ? vpnName
 577                                 : rd, prefix));
 578                 List<String> adjNextHop = nextHop.getNextHopIpList();
 579                 value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIpList(
 580                         (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : Arrays.asList(nextHopIp))
 581                         .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 582
 583                 if (nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 584                     LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, interfaceName, vpnName);
 585                     writeOperTxn.merge(
 586                             LogicalDatastoreType.OPERATIONAL,
 587                             VpnUtil.getPrefixToInterfaceIdentifier(
 588                                     VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 589                             VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix), true);
 590                 } else {
 591                     //Extra route adjacency
 592                     LOG.trace("Adding prefix {} and nexthopList {} as extra-route for vpn", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName);
 593                     writeOperTxn.merge(
 594                             LogicalDatastoreType.OPERATIONAL,
 595                             VpnUtil.getVpnToExtrarouteIdentifier(
 596                                     (rd != null) ? rd : vpnName, nextHop.getIpAddress()),
 597                             VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()), true);
 598                 }
 599             }
 600
 601             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 602
 603             VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 604             InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 605             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, true);
 606             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 607
 608             for (Adjacency nextHop : aug.getAdjacency()) {
 609                 long label = nextHop.getLabel();
 610                 if (rd != null) {
 611                     addToLabelMapper(label, dpnId, nextHop.getIpAddress(), Arrays.asList(nextHopIp), vpnId,
 612                             interfaceName, null,false, rd, writeOperTxn);
 613                     addPrefixToBGP(rd, nextHop.getIpAddress(), nextHopIp, label, writeConfigTxn);
 614                     //TODO: ERT - check for VPNs importing my route
 615                     for (VpnInstance vpn : vpnsToImportRoute) {
 616                         String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
 617                         if (vpnRd != null) {
 618                             LOG.debug("Exporting route with rd {} prefix {} nexthop {} label {} to VPN {}", vpnRd, nextHop.getIpAddress(), nextHopIp, label, vpn);
 619                             fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), Arrays.asList(nextHopIp), (int) label,
 620                                     RouteOrigin.SELF_IMPORTED, writeConfigTxn);
 621                         }
 622                     }
 623                 } else {
 624                     // ### add FIB route directly
 625                     fibManager.addOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), Arrays.asList(nextHopIp),
 626                             (int) label, RouteOrigin.STATIC, writeConfigTxn);
 627                 }
 628             }
 629         }
 630     }
 631
 632     private List<VpnInstance> getVpnsImportingMyRoute(final String vpnName) {
 633         List<VpnInstance> vpnsToImportRoute = new ArrayList<>();
 634
 635         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 636                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 637         Optional<VpnInstance> optVpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 638         final VpnInstance vpnInstance;
 639         if (optVpnInstance.isPresent()) {
 640             vpnInstance = optVpnInstance.get();
 641         } else {
 642             LOG.debug("Could not retrieve vpn instance {} to check for vpns importing the routes", vpnName);
 643             return vpnsToImportRoute;
 644         }
 645
 646         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 647             @Override
 648             public boolean apply(VpnInstance input) {
 649                 return !input.getVpnInstanceName().equals(vpnName);
 650             }
 651         };
 652
 653         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 654             @Override
 655             public boolean apply(VpnInstance input) {
 656                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ExportExtcommunity),
 657                         getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 658                 return Iterators.size(commonRTs.iterator()) > 0;
 659             }
 660         };
 661
 662         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 663             @Override
 664             public String apply(VpnInstance vpnInstance) {
 665                 //return vpnInstance.getVpnInstanceName();
 666                 return vpnInstance.getIpv4Family().getRouteDistinguisher();
 667             }
 668         };
 669
 670         vpnsToImportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(dataBroker)).
 671                 filter(excludeVpn).
 672                 filter(matchRTs).toList();
 673         return vpnsToImportRoute;
 674     }
 675
 676     private List<VpnInstance> getVpnsExportingMyRoute(final String vpnName) {
 677         List<VpnInstance> vpnsToExportRoute = new ArrayList<>();
 678
 679         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 680                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 681         Optional<VpnInstance> optVpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 682         final VpnInstance vpnInstance;
 683         if (optVpnInstance.isPresent()) {
 684             vpnInstance = optVpnInstance.get();
 685         } else {
 686             LOG.debug("Could not retrieve vpn instance {} to check for vpns exporting the routes", vpnName);
 687             return vpnsToExportRoute;
 688         }
 689
 690         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 691             @Override
 692             public boolean apply(VpnInstance input) {
 693                 return !input.getVpnInstanceName().equals(vpnName);
 694             }
 695         };
 696
 697         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 698             @Override
 699             public boolean apply(VpnInstance input) {
 700                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ImportExtcommunity),
 701                         getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
 702                 return Iterators.size(commonRTs.iterator()) > 0;
 703             }
 704         };
 705
 706         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 707             @Override
 708             public String apply(VpnInstance vpnInstance) {
 709                 return vpnInstance.getVpnInstanceName();
 710             }
 711         };
 712
 713         vpnsToExportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(dataBroker)).
 714                 filter(excludeVpn).
 715                 filter(matchRTs).toList();
 716         return vpnsToExportRoute;
 717     }
 718
 719     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
 720         final Predicate<T> inPredicate = Predicates.<T>in(collection2);
 721         return new Iterable<T>() {
 722             @Override
 723             public Iterator<T> iterator() {
 724                 return Iterators.filter(collection1.iterator(), inPredicate);
 725             }
 726         };
 727     }
 728
 729     private List<String> getRts(VpnInstance vpnInstance, VpnTarget.VrfRTType rtType) {
 730         String name = vpnInstance.getVpnInstanceName();
 731         List<String> rts = new ArrayList<>();
 732         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 733         if (vpnConfig == null) {
 734             LOG.trace("vpn config is not available for {}", name);
 735             return rts;
 736         }
 737         VpnTargets targets = vpnConfig.getVpnTargets();
 738         if (targets == null) {
 739             LOG.trace("vpn targets not available for {}", name);
 740             return rts;
 741         }
 742         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 743         if (vpnTargets == null) {
 744             LOG.trace("vpnTarget values not available for {}", name);
 745             return rts;
 746         }
 747         for (VpnTarget target : vpnTargets) {
 748             //TODO: Check for RT type is Both
 749             if(target.getVrfRTType().equals(rtType) ||
 750                     target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
 751                 String rtValue = target.getVrfRTValue();
 752                 rts.add(rtValue);
 753             }
 754         }
 755         return rts;
 756     }
 757
 758     private List<String> getExportRts(VpnInstance vpnInstance) {
 759         List<String> exportRts = new ArrayList<>();
 760         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 761         VpnTargets targets = vpnConfig.getVpnTargets();
 762         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 763         for (VpnTarget target : vpnTargets) {
 764             if (target.getVrfRTType().equals(VpnTarget.VrfRTType.ExportExtcommunity)) {
 765                 String rtValue = target.getVrfRTValue();
 766                 exportRts.add(rtValue);
 767             }
 768         }
 769         return exportRts;
 770     }
 771
 772     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 773                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow,
 774                              WriteTransaction writeConfigTxn){
 775         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 776         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, MetaDataUtil.getVpnIdMetadata(vpnId));
 777         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 778                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 779
 780         // Matching Arp reply flows
 781         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 782         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 783                 metadata, metadataMask }));
 784
 785         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 786
 787         // Instruction to punt to controller
 788         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 789         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 790         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 791         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
 792                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
 793
 794         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 795
 796         // Install the flow entry in L3_INTERFACE_TABLE
 797         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 798                 NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 799         FlowEntity flowEntity;
 800         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 801                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 802                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 803
 804         Flow flow = flowEntity.getFlowBuilder().build();
 805         String flowId = flowEntity.getFlowId();
 806         FlowKey flowKey = new FlowKey( new FlowId(flowId));
 807         Node nodeDpn = buildDpnNode(dpId);
 808
 809         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 810                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 811                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 812
 813         if (writeConfigTxn != null) {
 814             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 815                 LOG.debug("Creating ARP Flow for interface {}", vpnInterfaceName);
 816                 writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
 817             } else {
 818                 LOG.debug("Deleting ARP Flow for interface {}", vpnInterfaceName);
 819                 writeConfigTxn.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 820             }
 821         } else {
 822             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 823                 LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 824                 mdsalManager.installFlow(flowEntity);
 825             } else {
 826                 LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 827                 mdsalManager.removeFlow(flowEntity);
 828             }
 829         }
 830     }
 831
 832     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
 833     private Node buildDpnNode(BigInteger dpnId) {
 834         NodeId nodeId = new NodeId("openflow:" + dpnId);
 835         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
 836
 837         return nodeDpn;
 838     }
 839
 840     private String getRouteDistinguisher(String vpnName) {
 841         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 842                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 843         Optional<VpnInstance> vpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 844         String rd = null;
 845         if(vpnInstance.isPresent()) {
 846             VpnInstance instance = vpnInstance.get();
 847             VpnAfConfig config = instance.getIpv4Family();
 848             rd = config.getRouteDistinguisher();
 849         }
 850         return rd;
 851     }
 852
 853     /**
 854      * JobCallback class is used as a future callback for
 855      * main and rollback workers to handle success and failure.
 856      */
 857     private class DpnEnterExitVpnWorker implements FutureCallback<List<Void>> {
 858         BigInteger dpnId;
 859         String vpnName;
 860         String rd;
 861         boolean entered;
 862
 863         public DpnEnterExitVpnWorker(BigInteger dpnId, String vpnName, String rd, boolean entered) {
 864             this.entered = entered;
 865             this.dpnId = dpnId;
 866             this.vpnName = vpnName;
 867             this.rd = rd;
 868         }
 869
 870         /**
 871          * @param voids
 872          * This implies that all the future instances have returned success. -- TODO: Confirm this
 873          */
 874         @Override
 875         public void onSuccess(List<Void> voids) {
 876             if (entered) {
 877                 publishAddNotification(dpnId, vpnName, rd);
 878             } else {
 879                 publishRemoveNotification(dpnId, vpnName, rd);
 880             }
 881         }
 882
 883         /**
 884          *
 885          * @param throwable
 886          * This method is used to handle failure callbacks.
 887          * If more retry needed, the retrycount is decremented and mainworker is executed again.
 888          * After retries completed, rollbackworker is executed.
 889          * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 890          */
 891         @Override
 892         public void onFailure(Throwable throwable) {
 893             LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 894         }
 895     }
 896
 897     private void createOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 898         String routeDistinguisher = getRouteDistinguisher(vpnName);
 899         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 900         Boolean newDpnOnVpn = Boolean.FALSE;
 901
 902         synchronized (vpnName.intern()) {
 903             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 904             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 905             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 906             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data
 907                     .entry.vpn.to.dpn.list.VpnInterfaces
 908                     vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 909
 910             if (dpnInVpn.isPresent()) {
 911                 VpnToDpnList vpnToDpnList = dpnInVpn.get();
 912                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 913                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = vpnToDpnList.getVpnInterfaces();
 914                 if (vpnInterfaces == null) {
 915                     vpnInterfaces = new ArrayList<>();
 916                 }
 917                 vpnInterfaces.add(vpnInterface);
 918                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder(vpnToDpnList);
 919                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 920
 921                 if (writeTxn != null) {
 922                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 923                 } else {
 924                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 925                 }
 926                 /* If earlier state was inactive, it is considered new DPN coming back to the
 927                  * same VPN
 928                  */
 929                 if (vpnToDpnList.getDpnState() == VpnToDpnList.DpnState.Inactive) {
 930                     newDpnOnVpn = Boolean.TRUE;
 931                 }
 932             } else {
 933                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 934                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = new ArrayList<>();
 935                 vpnInterfaces.add(vpnInterface);
 936                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder().setDpnId(dpnId);
 937                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 938
 939                 if (writeTxn != null) {
 940                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 941                 } else {
 942                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 943                 }
 944                 newDpnOnVpn = Boolean.TRUE;
 945             }
 946             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
 947             try {
 948                 futures.get();
 949             } catch (InterruptedException | ExecutionException e) {
 950                 LOG.error("Error adding to dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
 951                 throw new RuntimeException(e.getMessage());
 952             }
 953         }
 954         /*
 955          * Informing the Fib only after writeTxn is submitted successfuly.
 956          */
 957         if (newDpnOnVpn) {
 958             LOG.debug("Sending populateFib event for new dpn {} in VPN {}", dpnId, vpnName);
 959             fibManager.populateFibOnNewDpn(dpnId, vpnId, rd, new DpnEnterExitVpnWorker(dpnId, vpnName, rd, true /* entered */));
 960         }
 961     }
 962
 963     private void removeOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 964         Boolean lastDpnOnVpn = Boolean.FALSE;
 965         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
 966         synchronized (vpnName.intern()) {
 967             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 968             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 969             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 970             if (dpnInVpn.isPresent()) {
 971                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 972                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 973                 org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 974                         currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 975
 976                 if (vpnInterfaces.remove(currVpnInterface)) {
 977                     if (vpnInterfaces.isEmpty()) {
 978                         List<IpAddresses> ipAddresses = dpnInVpn.get().getIpAddresses();
 979                         if (ipAddresses == null || ipAddresses.isEmpty()) {
 980                             VpnToDpnListBuilder dpnInVpnBuilder =
 981                                     new VpnToDpnListBuilder(dpnInVpn.get())
 982                                             .setDpnState(VpnToDpnList.DpnState.Inactive)
 983                                             .setVpnInterfaces(null);
 984                             if (writeTxn != null) {
 985                                 writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build(), true);
 986                             } else {
 987                                 VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build());
 988                             }
 989                             lastDpnOnVpn = Boolean.TRUE;
 990                         } else {
 991                             LOG.warn("vpn interfaces are empty but ip addresses are present for the vpn {} in dpn {}", vpnName, dpnId);
 992                         }
 993                     } else {
 994                         if (writeTxn != null) {
 995                             writeTxn.delete(LogicalDatastoreType.OPERATIONAL, id.child(
 996                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 997                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 998                                     new VpnInterfacesKey(intfName)));
 999                         } else {
1000                             VpnUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1001                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1002                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1003                                     new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
1004                         }
1005                     }
1006                 }
1007             }
1008             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
1009             try {
1010                 futures.get();
1011             } catch (InterruptedException | ExecutionException e) {
1012                 LOG.error("Error removing from dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
1013                 throw new RuntimeException(e.getMessage());
1014             }
1015         }
1016         if (lastDpnOnVpn) {
1017             LOG.debug("Sending cleanup event for dpn {} in VPN {}", dpnId, vpnName);
1018             fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd, new DpnEnterExitVpnWorker(dpnId, vpnName, rd, false /* exited */));
1019         }
1020     }
1021
1022     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1023         List<VpnInstance> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1024         for (VpnInstance vpn : vpnsToExportRoute) {
1025             String rd = vpn.getIpv4Family().getRouteDistinguisher();
1026             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getIpv4Family().getRouteDistinguisher());
1027             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1028             if (vrfEntries != null) {
1029                 for (VrfEntry vrfEntry : vrfEntries) {
1030                     try {
1031                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1032                             continue;
1033                         }
1034                         String prefix = vrfEntry.getDestPrefix();
1035                         long label = vrfEntry.getLabel();
1036                         List<String> nextHops = vrfEntry.getNextHopAddressList();
1037                         SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1038                         for (String nh : nextHops) {
1039                             if (route != null) {
1040                                 LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1041                                 importSubnetRouteForNewVpn(rd, prefix, nh, (int)label, route, writeConfigTxn);
1042                             } else {
1043                                 LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1044                                 fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, prefix, Arrays.asList(nh), (int)label,
1045                                         RouteOrigin.SELF_IMPORTED, writeConfigTxn);
1046                             }
1047                         }
1048                     } catch (Exception e) {
1049                         LOG.error("Exception occurred while importing route with prefix {} label {} nexthop {} from vpn {} to vpn {}", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpn.getVpnInstanceName(), vpnName);
1050                     }
1051                 }
1052                 writeConfigTxn.submit();
1053             } else {
1054                 LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getIpv4Family().getRouteDistinguisher());
1055             }
1056         }
1057     }
1058
1059     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label, WriteTransaction writeConfigTxn) {
1060         try {
1061             LOG.info("ADD: Adding Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1062             fibManager.addOrUpdateFibEntry(dataBroker, rd, prefix, Arrays.asList(nextHopIp), (int)label, RouteOrigin.STATIC, writeConfigTxn);
1063             bgpManager.advertisePrefix(rd, prefix, Arrays.asList(nextHopIp), (int)label);
1064             LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1065         } catch(Exception e) {
1066             LOG.error("Add prefix failed", e);
1067         }
1068     }
1069
1070     @Override
1071     public void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1072         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
1073         LOG.info("VPN Interface remove event - intfName {}" ,vpnInterface.getName());
1074         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1075         final String interfaceName = key.getName();
1076
1077         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1078         final Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1079         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
1080                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1081         if (optVpnInterface.isPresent()){
1082             BigInteger dpnId = BigInteger.ZERO;
1083             Boolean dpnIdRetrieved = Boolean.FALSE;
1084             if(interfaceState != null){
1085                 try{
1086                     dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1087                     dpnIdRetrieved = Boolean.TRUE;
1088                 }catch (Exception e){
1089                     LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", interfaceName, e);
1090                 }
1091             } else {
1092                 LOG.error("Unable to retrieve interfaceState for interface {} , quitting ", interfaceName);
1093                 return;
1094             }
1095             final VpnInterface vpnOpInterface = optVpnInterface.get();
1096             if(dpnIdRetrieved == Boolean.FALSE){
1097                 LOG.info("dpnId for {} has not been retrieved yet. Fetching from vpn interface operational DS", interfaceName);
1098                 dpnId = vpnOpInterface.getDpnId();
1099             }
1100             final int ifIndex = interfaceState.getIfIndex();
1101             final BigInteger dpId = dpnId;
1102             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1103             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1104                     new Callable<List<ListenableFuture<Void>>>() {
1105                         @Override
1106                         public List<ListenableFuture<Void>> call() throws Exception {
1107                             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1108                             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1109                             WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
1110                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, false, true, writeConfigTxn, writeOperTxn, writeInvTxn);
1111                             List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
1112                             futures.add(writeOperTxn.submit());
1113                             futures.add(writeConfigTxn.submit());
1114                             futures.add(writeInvTxn.submit());
1115                             return futures;
1116                         }
1117                     });
1118
1119         }else{
1120             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
1121         }
1122     }
1123
1124     protected void processVpnInterfaceDown(BigInteger dpId,
1125                                            String interfaceName,
1126                                            int lPortTag,
1127                                            boolean isInterfaceStateDown,
1128                                            boolean isConfigRemoval,
1129                                            WriteTransaction writeConfigTxn,
1130                                            WriteTransaction writeOperTxn,
1131                                            WriteTransaction writeInvTxn) {
1132         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1133         if (!isInterfaceStateDown) {
1134             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1135             if(vpnInterface == null){
1136                 LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
1137                 return;
1138             }else{
1139                 final String vpnName = vpnInterface.getVpnInstanceName();
1140                 if(!vpnInterface.isScheduledForRemove()){
1141                     VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE, writeOperTxn);
1142                     removeAdjacenciesFromVpn(dpId, interfaceName, vpnInterface.getVpnInstanceName(), writeConfigTxn);
1143                     LOG.info("Unbinding vpn service from interface {} ", interfaceName);
1144                     unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown, isConfigRemoval, writeConfigTxn, writeInvTxn);
1145                 }else{
1146                     LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ", interfaceName);
1147                     return;
1148                 }
1149             }
1150         } else {
1151             // Interface is retained in the DPN, but its Link Down.
1152             // Only withdraw the prefixes for this interface from BGP
1153             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1154             if(vpnInterface == null){
1155                 LOG.info("Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in operational data store", interfaceName);
1156                 return;
1157             }else {
1158                 withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
1159             }
1160         }
1161     }
1162
1163     private void waitForFibToRemoveVpnPrefix(String interfaceName) {
1164         // FIB didn't get a chance yet to clean up this VPNInterface
1165         // Let us give it a chance here !
1166         LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
1167         try {
1168             Runnable notifyTask = new VpnNotifyTask();
1169             vpnIntfMap.put(interfaceName, notifyTask);
1170             synchronized (notifyTask) {
1171                 try {
1172                     notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
1173                 } catch (InterruptedException e) {
1174                 }
1175             }
1176         } finally {
1177             vpnIntfMap.remove(interfaceName);
1178         }
1179     }
1180
1181     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final String interfaceName, final String vpnName,
1182                                           WriteTransaction writeConfigTxn) {
1183         //Read NextHops
1184         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1185         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1186         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1187
1188         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1189         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", interfaceName,
1190                 vpnName, rd);
1191         if (adjacencies.isPresent()) {
1192             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1193
1194             if (!nextHops.isEmpty()) {
1195                 LOG.trace("NextHops are " + nextHops);
1196                 for (Adjacency nextHop : nextHops) {
1197                     List<String> nhList = new ArrayList<String>();
1198                     if (nextHop.getMacAddress() == null || nextHop.getMacAddress().isEmpty()) {
1199                         // This is either an extra-route (or) a learned IP via subnet-route
1200                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1201                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1202                             LOG.error("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
1203                                     rd, nextHop.getIpAddress());
1204                             continue;
1205                         }
1206                         nhList = Arrays.asList(nextHopIp);
1207                     } else {
1208                         // This is a primary adjacency
1209                         nhList = nextHop.getNextHopIpList();
1210                     }
1211                     if (rd.equals(vpnName)) {
1212                         //this is an internal vpn - the rd is assigned to the vpn instance name;
1213                         //remove from FIB directly
1214                         for(String nh : nhList) {
1215                             fibManager.removeOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), nh, writeConfigTxn);
1216                         }
1217                     } else {
1218                         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1219                         for (String nh : nhList) {
1220                             //IRT: remove routes from other vpns importing it
1221                             removePrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1222                             for (VpnInstance vpn : vpnsToImportRoute) {
1223                                 String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
1224                                 if (vpnRd != null) {
1225                                     LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd, nextHop.getIpAddress(), vpn.getVpnInstanceName());
1226                                     fibManager.removeOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), nh, writeConfigTxn);
1227                                 }
1228                             }
1229                         }
1230                     }
1231                     String ip = nextHop.getIpAddress().split("/")[0];
1232                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
1233                             vpnName, ip);
1234                     if (vpnPortipToPort != null && !vpnPortipToPort.isConfig()) {
1235                         LOG.trace("VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
1236                                 vpnPortipToPort.getPortName(),ip);
1237                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip);
1238                     }
1239                 }
1240             }
1241         }
1242     }
1243
1244
1245     private void unbindService(BigInteger dpId, String vpnInstanceName, final String vpnInterfaceName,
1246                                int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval,
1247                                WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
1248         short l3vpn_service_index = ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX);
1249         if (!isInterfaceStateDown && isConfigRemoval) {
1250             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1251             dataStoreCoordinator.enqueueJob(vpnInterfaceName,
1252                     new Callable<List<ListenableFuture<Void>>>() {
1253                         @Override
1254                         public List<ListenableFuture<Void>> call() throws Exception {
1255                             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
1256                             writeTxn.delete(LogicalDatastoreType.CONFIGURATION,
1257                                     InterfaceUtils.buildServiceId(vpnInterfaceName,
1258                                             NwConstants.L3VPN_SERVICE_INDEX));
1259
1260                             List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
1261                             futures.add(writeTxn.submit());
1262                             return futures;
1263                         }
1264                     });
1265         }
1266         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
1267         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
1268                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW, writeInvTxn);
1269         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
1270                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.DEL_FLOW, writeInvTxn);
1271     }
1272
1273
1274     private void removePrefixFromBGP(String rd, String prefix, String nextHop, WriteTransaction writeConfigTxn) {
1275         try {
1276             LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
1277             fibManager.removeOrUpdateFibEntry(dataBroker, rd, prefix, nextHop, writeConfigTxn);
1278             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1279             LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
1280         } catch(Exception e) {
1281             LOG.error("Delete prefix failed", e);
1282         }
1283     }
1284
1285     @Override
1286     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
1287         LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
1288         LOG.info("VPN Interface update event - intfName {}" ,update.getName());
1289         String oldVpnName = original.getVpnInstanceName();
1290         String newVpnName = update.getVpnInstanceName();
1291         BigInteger dpnId = update.getDpnId();
1292         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
1293         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
1294         if (oldAdjs == null) {
1295             oldAdjs = new ArrayList<>();
1296         }
1297         if (newAdjs == null) {
1298             newAdjs = new ArrayList<>();
1299         }
1300         //handles switching between <internal VPN - external VPN>
1301         if (!oldVpnName.equals(newVpnName)) {
1302             remove(identifier, original);
1303             waitForFibToRemoveVpnPrefix(update.getName());
1304             add(identifier, update);
1305         }
1306         //handle both addition and removal of adjacencies
1307         //currently, new adjacency may be an extra route
1308         if (!oldAdjs.equals(newAdjs)) {
1309             for (Adjacency adj : newAdjs) {
1310                 if (oldAdjs.contains(adj)) {
1311                     oldAdjs.remove(adj);
1312                 } else {
1313                     // add new adjacency - right now only extra route will hit this path
1314                     addNewAdjToVpnInterface(identifier, adj, dpnId);
1315                 }
1316             }
1317             for (Adjacency adj : oldAdjs) {
1318                 delAdjFromVpnInterface(identifier, adj, dpnId);
1319             }
1320         }
1321     }
1322
1323     public void processArpRequest(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715
1324                                           .IpAddress srcIP, PhysAddress srcMac, org.opendaylight.yang.gen.v1.urn.ietf
1325             .params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress targetIP, PhysAddress targetMac, String srcInterface){
1326         //Build ARP response with ARP requests TargetIp TargetMac as the Arp Response SrcIp and SrcMac
1327         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
1328                 .setDstIpaddress(srcIP).setDstMacaddress(srcMac).setSrcIpaddress(targetIP).setSrcMacaddress(targetMac).build();
1329         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
1330         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
1331         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
1332             @Override
1333             public void onFailure(Throwable error) {
1334                 LOG.error("Error - {}", msgFormat, error);
1335             }
1336
1337             @Override
1338             public void onSuccess(RpcResult<Void> result) {
1339                 if(!result.isSuccessful()) {
1340                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
1341                 } else {
1342                     LOG.debug("Successful RPC Result - {}", msgFormat);
1343                 }
1344             }
1345         });
1346     }
1347
1348     private String getErrorText(Collection<RpcError> errors) {
1349         StringBuilder errorText = new StringBuilder();
1350         for(RpcError error : errors) {
1351             errorText.append(",").append(error.getErrorType()).append("-")
1352                     .append(error.getMessage());
1353         }
1354         return errorText.toString();
1355     }
1356
1357     private void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1358                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd,
1359                                   WriteTransaction writeOperTxn) {
1360         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1361         Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
1362         Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
1363         Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
1364         if (!isSubnetRoute) {
1365             // NextHop must be present for non-subnetroute entries
1366             Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
1367         }
1368         LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
1369         if (dpnId != null) {
1370             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1371                     .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
1372             LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1373             lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(vpnId)
1374                     .setIsSubnetRoute(isSubnetRoute);
1375             if (elanTag != null) {
1376                 lriBuilder.setElanTag(elanTag);
1377             }
1378             if (vpnInterfaceName != null) {
1379                 lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1380             }
1381             lriBuilder.setParentVpnRd(rd);
1382             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
1383             if (vpnInstanceOpDataEntry != null) {
1384                 List<String> vpnInstanceNames = Arrays.asList(vpnInstanceOpDataEntry.getVpnInstanceName());
1385                 lriBuilder.setVpnInstanceList(vpnInstanceNames);
1386             }
1387             LabelRouteInfo lri = lriBuilder.build();
1388             LOG.trace("Adding route info to label map: {}", lri);
1389             if (writeOperTxn != null) {
1390                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, lriIid, lri, true);
1391             } else {
1392                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
1393             }
1394         } else {
1395             LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
1396         }
1397     }
1398
1399     public synchronized void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
1400                                                         long elantag, BigInteger dpnId, WriteTransaction writeTxn) {
1401         SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
1402         RouteOrigin origin = RouteOrigin.STATIC; // Only case when a route is considered as directly connected
1403         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1404                 .setLabel((long)label).setOrigin(origin.getValue())
1405                 .addAugmentation(SubnetRoute.class, route).build();
1406
1407         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
1408
1409         //TODO: What should be parentVpnId? Get it from RD?
1410         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1411         addToLabelMapper((long)label, dpnId, prefix, Arrays.asList(nextHop), vpnId, null, elantag, true, rd, null);
1412         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1413
1414         InstanceIdentifierBuilder<VrfTables> idBuilder =
1415                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1416         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1417
1418         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1419                 setVrfEntry(vrfEntryList).build();
1420
1421         if (writeTxn != null) {
1422             writeTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1423         } else {
1424             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1425         }
1426
1427         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1428         if (vpnsToImportRoute.size() > 0) {
1429             VrfEntry importingVrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1430                     .setLabel((long) label).setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
1431                     .addAugmentation(SubnetRoute.class, route).build();
1432             List<VrfEntry> importingVrfEntryList = Arrays.asList(importingVrfEntry);
1433             for (VpnInstance vpnInstance : vpnsToImportRoute) {
1434                 LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop, label, vpnInstance.getVpnInstanceName());
1435                 String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1436                 InstanceIdentifier<VrfTables> importingVrfTableId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(importingRd)).build();
1437                 VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(importingVrfEntryList).build();
1438                 if (writeTxn != null) {
1439                     writeTxn.merge(LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable, true);
1440                 } else {
1441                     VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable);
1442                 }
1443             }
1444         }
1445     }
1446
1447     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1448                                                         SubnetRoute route, WriteTransaction writeConfigTxn) {
1449
1450         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1451         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1452                 .setLabel((long)label).setOrigin(origin.getValue())
1453                 .addAugmentation(SubnetRoute.class, route).build();
1454         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, route.getElantag());
1455         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1456         InstanceIdentifierBuilder<VrfTables> idBuilder =
1457                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1458         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1459         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1460                 setVrfEntry(vrfEntryList).build();
1461         if (writeConfigTxn != null) {
1462             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1463         } else {
1464             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1465         }
1466     }
1467
1468     public synchronized void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName){
1469         fibManager.removeFibEntry(dataBroker, rd, prefix, null);
1470         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1471         for (VpnInstance vpnInstance : vpnsToImportRoute) {
1472             String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1473             LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix, vpnInstance.getVpnInstanceName());
1474             fibManager.removeFibEntry(dataBroker, importingRd, prefix, null);
1475         }
1476     }
1477
1478     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1479
1480         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1481
1482         if (optVpnInterface.isPresent()) {
1483             VpnInterface currVpnIntf = optVpnInterface.get();
1484             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1485             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1486             rd = (rd != null) ? rd : currVpnIntf.getVpnInstanceName();
1487             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1488             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, adjPath);
1489             long label =
1490                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1491                             VpnUtil.getNextHopLabelKey(rd, prefix));
1492
1493             List<Adjacency> adjacencies;
1494             if (optAdjacencies.isPresent()) {
1495                 adjacencies = optAdjacencies.get().getAdjacency();
1496             } else {
1497                 //This code will not be hit since VM adjacency will always be there
1498                 adjacencies = new ArrayList<>();
1499             }
1500
1501             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIpList(adj.getNextHopIpList())
1502                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
1503
1504             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1505             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove());
1506
1507             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1508             for (String nh : adj.getNextHopIpList()) {
1509                 addExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(), (int) label,
1510                         currVpnIntf.getName());
1511             }
1512         }
1513     }
1514
1515     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1516         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1517
1518         if (optVpnInterface.isPresent()) {
1519             VpnInterface currVpnIntf = optVpnInterface.get();
1520
1521             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1522             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1523             if (optAdjacencies.isPresent()) {
1524                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1525
1526                 if (!adjacencies.isEmpty()) {
1527                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1528                     rd = (rd != null) ? rd :currVpnIntf.getVpnInstanceName();
1529                     LOG.trace("Adjacencies are " + adjacencies);
1530                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1531                     while (adjIt.hasNext()) {
1532                         Adjacency adjElem = adjIt.next();
1533                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1534                             adjIt.remove();
1535
1536                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1537                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1538                                     currVpnIntf.getVpnInstanceName(),
1539                                     aug, dpnId, currVpnIntf.isScheduledForRemove());
1540
1541                             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1542
1543                             for (String nh : adj.getNextHopIpList()) {
1544                                 delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
1545                                         currVpnIntf.getName());
1546                             }
1547                             break;
1548                         }
1549
1550                     }
1551                 }
1552             }
1553         }
1554
1555     }
1556
1557     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label,
1558                                  String intfName) {
1559
1560         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1561         VpnUtil.syncUpdate(
1562                 dataBroker,
1563                 LogicalDatastoreType.OPERATIONAL,
1564                 VpnUtil.getVpnToExtrarouteIdentifier( (rd != null) ? rd : routerID, destination),
1565                 VpnUtil.getVpnToExtraroute(destination, Arrays.asList(nextHop)));
1566
1567         BigInteger dpnId = null;
1568         if (intfName != null && !intfName.isEmpty()) {
1569             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1570             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1571             if (nextHopIp == null || nextHopIp.isEmpty()) {
1572                 LOG.error("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1573                         intfName, destination);
1574                 return;
1575             }
1576             nextHop = nextHopIp;
1577         }
1578         List<String> nextHopIpList = Arrays.asList(nextHop);
1579         if (rd != null) {
1580             /* Label mapper is required only for BGP VPN and not for Internal VPN */
1581             addToLabelMapper((long) label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(dataBroker, routerID),
1582                     intfName, null, false, rd, null);
1583         }
1584
1585         // TODO (eperefr): This is a limitation to be stated in docs. When configuring static route to go to
1586         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1587         // first place.
1588         InterVpnLink interVpnLink = VpnUtil.getInterVpnLinkByEndpointIp(dataBroker, nextHop);
1589         if ( interVpnLink != null ) {
1590             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1591             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1592             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1593             // This is like leaking one of the Vpn2 routes towards Vpn1
1594             boolean nexthopIsVpn2 = ( interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nextHop) );
1595             String srcVpnUuid = (nexthopIsVpn2) ? interVpnLink.getSecondEndpoint().getVpnUuid().getValue()
1596                     : interVpnLink.getFirstEndpoint().getVpnUuid().getValue();
1597             String dstVpnUuid = (nexthopIsVpn2) ? interVpnLink.getFirstEndpoint().getVpnUuid().getValue()
1598                     : interVpnLink.getSecondEndpoint().getVpnUuid().getValue();
1599             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
1600             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1601                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1602             VpnUtil.leakRoute(dataBroker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel);
1603         } else {
1604             if (rd != null) {
1605                 addPrefixToBGP(rd, destination, nextHop, label, null);
1606             } else {
1607                 // ### add FIB route directly
1608                 fibManager.addOrUpdateFibEntry(dataBroker, routerID, destination, Arrays.asList(nextHop), label, RouteOrigin.STATIC, null);
1609             }
1610         }
1611     }
1612
1613     protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName) {
1614         if (intfName != null && !intfName.isEmpty()) {
1615             BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1616             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1617             if (nextHopIp == null || nextHopIp.isEmpty()) {
1618                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1619                         intfName, destination);
1620             }
1621             nextHop = nextHopIp;
1622         }
1623
1624         if (rd != null) {
1625             removePrefixFromBGP(rd, destination, nextHop, null);
1626         } else {
1627             // ### add FIB route directly
1628             fibManager.removeOrUpdateFibEntry(dataBroker, routerID, destination, nextHop, null);
1629         }
1630     }
1631
1632     void publishAddNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1633         LOG.debug("Sending notification for add dpn {} in vpn {} event ", dpnId, vpnName);
1634         AddEventData data = new AddEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1635         AddDpnEvent event = new AddDpnEventBuilder().setAddEventData(data).build();
1636         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1637         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1638             @Override
1639             public void onFailure(Throwable error) {
1640                 LOG.warn("Error in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName, error);
1641             }
1642
1643             @Override
1644             public void onSuccess(Object arg) {
1645                 LOG.trace("Successful in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName);
1646             }
1647         });
1648     }
1649
1650     void publishRemoveNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1651         LOG.debug("Sending notification for remove dpn {} in vpn {} event ", dpnId, vpnName);
1652         RemoveEventData data = new RemoveEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1653         RemoveDpnEvent event = new RemoveDpnEventBuilder().setRemoveEventData(data).build();
1654         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1655         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1656             @Override
1657             public void onFailure(Throwable error) {
1658                 LOG.warn("Error in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName, error);
1659             }
1660
1661             @Override
1662             public void onSuccess(Object arg) {
1663                 LOG.trace("Successful in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName);
1664             }
1665         });
1666     }
1667
1668     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1669         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1670                 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1671                 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1672     }
1673
1674     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1675         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1676                 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1677     }
1678
1679     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1680         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1681         if(dpId.equals(BigInteger.ZERO)) {
1682             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
1683             return;
1684         }
1685         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1686
1687         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1688                 .OPERATIONAL, routerDpnListIdentifier);
1689         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1690         if (optionalRouterDpnList.isPresent()) {
1691             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1692                     RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1693         } else {
1694             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1695             builder.setRouterId(routerName);
1696             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1697             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
1698             routerInterfaces.add(routerInterface);
1699             builder.setDpnVpninterfacesList(Arrays.asList(dpnVpnList.build()));
1700             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1701                     getRouterId(routerName),
1702                     builder.build(), true);
1703         }
1704     }
1705
1706     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1707         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1708         if(dpId.equals(BigInteger.ZERO)) {
1709             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1710             return;
1711         }
1712         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1713         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1714                 .OPERATIONAL, routerDpnListIdentifier);
1715         if (optionalRouterDpnList.isPresent()) {
1716             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1717             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1718
1719             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1720                 if (routerInterfaces.isEmpty()) {
1721                     if (writeOperTxn != null) {
1722                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1723                     } else {
1724                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1725                     }
1726                 } else {
1727                     if (writeOperTxn != null) {
1728                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1729                                 RouterInterfaces.class,
1730                                 new RouterInterfacesKey(vpnInterfaceName)));
1731                     } else {
1732                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1733                                 RouterInterfaces.class,
1734                                 new RouterInterfacesKey(vpnInterfaceName)));
1735                     }
1736                 }
1737             }
1738         }
1739     }
1740
1741     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1742                                                   WriteTransaction writeOperTxn) {
1743         if(dpId.equals(BigInteger.ZERO)) {
1744             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1745             return;
1746         }
1747         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1748         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1749                 .OPERATIONAL, routerDpnListIdentifier);
1750         if (optionalRouterDpnList.isPresent()) {
1751             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1752             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1753             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1754                 if (routerInterfaces.isEmpty()) {
1755                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1756                 } else {
1757                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1758                             RouterInterfaces.class,
1759                             new RouterInterfacesKey(vpnInterfaceName)));
1760                 }
1761             }
1762         }
1763     }
1764
1765     public void addMIPAdjacency(String vpnName,String vpnInterface, org.opendaylight.yang.gen.v1.urn.ietf.params.xml
1766             .ns.yang.ietf.inet.types.rev130715.IpAddress prefix){
1767
1768         LOG.trace("Adding {} adjacency to VPN Interface {} ",prefix,vpnInterface);
1769         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
1770         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
1771         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
1772         String nextHopIpAddr = null;
1773         String nextHopMacAddress = null;
1774         String ip = prefix.getIpv4Address().getValue();
1775         if (adjacencies.isPresent()) {
1776             List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
1777             ip = VpnUtil.getIpPrefix(ip);
1778             for (Adjacency adjacs : adjacencyList) {
1779                 if (adjacs.getMacAddress() != null && !adjacs.getMacAddress().isEmpty()) {
1780                     nextHopIpAddr = adjacs.getIpAddress();
1781                     nextHopMacAddress = adjacs.getMacAddress();
1782                     break;
1783                 }
1784             }
1785             if (nextHopMacAddress != null && ip != null) {
1786                 String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1787                 long label =
1788                         VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1789                                 VpnUtil.getNextHopLabelKey((rd != null) ? rd : vpnName, ip));
1790                 String nextHopIp = nextHopIpAddr.split("/")[0];
1791                 Adjacency newAdj = new AdjacencyBuilder().setIpAddress(ip).setKey
1792                         (new AdjacencyKey(ip)).setNextHopIpList(Arrays.asList(nextHopIp)).build();
1793                 adjacencyList.add(newAdj);
1794                 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
1795                 VpnInterface newVpnIntf = new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(vpnInterface)).
1796                         setName(vpnInterface).setVpnInstanceName(vpnName).addAugmentation(Adjacencies.class, aug).build();
1797                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vpnIfId, newVpnIntf);
1798                 LOG.debug(" Successfully stored subnetroute Adjacency into VpnInterface {}", vpnInterface);
1799             }
1800         }
1801
1802     }
1803
1804     public void removeMIPAdjacency(String vpnName, String vpnInterface, org.opendaylight.yang.gen.v1.urn.ietf.params
1805             .xml.ns.yang.ietf.inet.types.rev130715.IpAddress prefix) {
1806         String ip = VpnUtil.getIpPrefix(prefix.getIpv4Address().getValue());
1807         LOG.trace("Removing {} adjacency from Old VPN Interface {} ",ip,vpnInterface);
1808         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
1809         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
1810         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1811         if (adjacencies.isPresent()) {
1812             InstanceIdentifier<Adjacency> adjacencyIdentifier = InstanceIdentifier.builder(VpnInterfaces.class).
1813                     child(VpnInterface.class, new VpnInterfaceKey(vpnInterface)).augmentation(Adjacencies.class)
1814                     .child(Adjacency.class, new AdjacencyKey(ip)).build();
1815             MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, adjacencyIdentifier);
1816             LOG.trace("Successfully Deleted Adjacency into VpnInterface {}", vpnInterface);
1817         }
1818     }
1819
1820     void notifyTaskIfRequired(String intfName) {
1821         Runnable notifyTask = vpnIntfMap.remove(intfName);
1822         if (notifyTask == null) {
1823             return;
1824         }
1825         executorService.execute(notifyTask);
1826     }
1827 }