vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Function;
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Preconditions;
  13 import com.google.common.base.Predicate;
  14 import com.google.common.base.Predicates;
  15 import com.google.common.collect.FluentIterable;
  16 import com.google.common.collect.Iterators;
  17 import com.google.common.util.concurrent.CheckedFuture;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.JdkFutureAdapters;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import java.math.BigInteger;
  23 import java.util.ArrayList;
  24 import java.util.Arrays;
  25 import java.util.Collection;
  26 import java.util.Iterator;
  27 import java.util.List;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ExecutionException;
  31 import java.util.concurrent.ExecutorService;
  32 import java.util.concurrent.Executors;
  33 import java.util.concurrent.Future;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  36 import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
  37 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  38 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  41 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  42 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  43 import org.opendaylight.genius.mdsalutil.ActionInfo;
  44 import org.opendaylight.genius.mdsalutil.ActionType;
  45 import org.opendaylight.genius.mdsalutil.FlowEntity;
  46 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  47 import org.opendaylight.genius.mdsalutil.InstructionType;
  48 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  49 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  50 import org.opendaylight.genius.mdsalutil.MatchInfo;
  51 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  52 import org.opendaylight.genius.mdsalutil.NwConstants;
  53 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  54 import org.opendaylight.genius.utils.ServiceIndex;
  55 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  56 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  57 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  58 import org.opendaylight.netvirt.vpnmanager.intervpnlink.InterVpnLinkUtil;
  59 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.VpnTargets;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  71 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInput;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInputBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEvent;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEventBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEvent;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEventBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventData;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventDataBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventData;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventDataBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 132 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 133 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 134 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 135 import org.opendaylight.yangtools.yang.common.RpcError;
 136 import org.opendaylight.yangtools.yang.common.RpcResult;
 137 import org.slf4j.Logger;
 138 import org.slf4j.LoggerFactory;
 139
 140 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
 141     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 142     private ListenerRegistration<DataChangeListener> listenerRegistration;
 143     private final DataBroker dataBroker;
 144     private final IBgpManager bgpManager;
 145     private final IFibManager fibManager;
 146     private final IMdsalApiManager mdsalManager;
 147     private final IdManagerService idManager;
 148     private final OdlArputilService arpManager;
 149     private final OdlInterfaceRpcService ifaceMgrRpcService;
 150     private final NotificationPublishService notificationPublishService;
 151     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
 152     private ExecutorService executorService = Executors.newSingleThreadExecutor();
 153
 154     public VpnInterfaceManager(final DataBroker dataBroker,
 155                                final IBgpManager bgpManager,
 156                                final OdlArputilService arpManager,
 157                                final IdManagerService idManager,
 158                                final IMdsalApiManager mdsalManager,
 159                                final IFibManager fibManager,
 160                                final OdlInterfaceRpcService ifaceMgrRpcService,
 161                                final NotificationPublishService notificationPublishService) {
 162         super(VpnInterface.class);
 163         this.dataBroker = dataBroker;
 164         this.bgpManager = bgpManager;
 165         this.arpManager = arpManager;
 166         this.idManager = idManager;
 167         this.mdsalManager = mdsalManager;
 168         this.fibManager = fibManager;
 169         this.ifaceMgrRpcService = ifaceMgrRpcService;
 170         this.notificationPublishService = notificationPublishService;
 171     }
 172
 173     public void start() {
 174         LOG.info("{} start", getClass().getSimpleName());
 175         listenerRegistration = dataBroker.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 176                 getWildCardPath(), this, DataChangeScope.SUBTREE);
 177     }
 178
 179     private InstanceIdentifier<VpnInterface> getWildCardPath() {
 180         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 181     }
 182
 183     @Override
 184     public void close() throws Exception {
 185         if (listenerRegistration != null) {
 186             listenerRegistration.close();
 187             listenerRegistration = null;
 188         }
 189         LOG.info("{} close", getClass().getSimpleName());
 190     }
 191
 192     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 193         return InstanceIdentifier.create(InterfacesState.class)
 194                 .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 195     }
 196
 197     @Override
 198     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 199         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 200         LOG.info("VPN Interface add event - intfName {}" ,vpnInterface.getName());
 201         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 202         final String interfaceName = key.getName();
 203
 204         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 205                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 206         if(interfaceState != null){
 207             try{
 208                 final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 209                 final int ifIndex = interfaceState.getIfIndex();
 210                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 211                 dataStoreCoordinator.enqueueJob("VPNINTERFACE-"+ interfaceName,
 212                         new Callable<List<ListenableFuture<Void>>>() {
 213                             @Override
 214                             public List<ListenableFuture<Void>> call() throws Exception {
 215                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 216                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 217                                 WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 218                                 processVpnInterfaceUp(dpnId, vpnInterface, ifIndex, false, writeConfigTxn, writeOperTxn, writeInvTxn);
 219                                 List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
 220                                 futures.add(writeOperTxn.submit());
 221                                 futures.add(writeConfigTxn.submit());
 222                                 futures.add(writeInvTxn.submit());
 223                                 return futures;
 224                             }
 225                         });
 226             }catch (Exception e){
 227                 LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", interfaceName, e);
 228                 return;
 229             }
 230         } else {
 231             LOG.info("Handling addition of VPN interface {} skipped as interfaceState is not available", interfaceName);
 232         }
 233     }
 234
 235     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface,
 236                                          final int lPortTag, boolean isInterfaceUp,
 237                                          WriteTransaction writeConfigTxn,
 238                                          WriteTransaction writeOperTxn,
 239                                          WriteTransaction writeInvTxn) {
 240
 241         final String interfaceName = vpnInterface.getName();
 242         if (!isInterfaceUp) {
 243             final String vpnName = vpnInterface.getVpnInstanceName();
 244             LOG.info("Binding vpn service to interface {} ", interfaceName);
 245             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 246             if (vpnId == VpnConstants.INVALID_ID) {
 247                 LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 248                 return;
 249             }
 250             boolean waitForVpnInterfaceOpRemoval = false;
 251             VpnInterface opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, vpnInterface.getName());
 252             if (opVpnInterface != null ) {
 253                 String opVpnName = opVpnInterface.getVpnInstanceName();
 254                 String primaryInterfaceIp = null;
 255                 if(opVpnName.equals(vpnName)) {
 256                     // Please check if the primary VRF Entry does not exist for VPNInterface
 257                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 258                     // back to back
 259                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 260                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 261                     if (adjs == null) {
 262                         LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained", interfaceName);
 263                         return;
 264                     }
 265                     for (Adjacency adj : adjs) {
 266                         if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
 267                             primaryInterfaceIp = adj.getIpAddress();
 268                             break;
 269                         }
 270                     }
 271                     if (primaryInterfaceIp == null) {
 272                         LOG.info("VPN Interface {} addition failed as primary adjacency "
 273                                 + "for this vpn interface could not be obtained", interfaceName);
 274                         return;
 275                     }
 276                     // Get the rd of the vpn instance
 277                     String rd = getRouteDistinguisher(opVpnName);
 278                     rd =  (rd == null) ? opVpnName : rd;
 279                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, rd, primaryInterfaceIp);
 280                     if (vrf != null) {
 281                         LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
 282                         return;
 283                     }
 284                     waitForVpnInterfaceOpRemoval = true;
 285                 } else {
 286                     LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
 287                             interfaceName, vpnName, opVpnName);
 288                 }
 289             }
 290             if (!waitForVpnInterfaceOpRemoval) {
 291                 // Add the VPNInterface and quit
 292                 updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 293                 bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 294                 processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 295                 return;
 296             }
 297
 298             // FIB didn't get a chance yet to clean up this VPNInterface
 299             // Let us give it a chance here !
 300             LOG.info("Trying to add VPN Interface {}, but waiting for FIB to clean up! ", interfaceName);
 301             try {
 302                 Runnable notifyTask = new VpnNotifyTask();
 303                 vpnIntfMap.put(interfaceName, notifyTask);
 304                 synchronized (notifyTask) {
 305                     try {
 306                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 307                     } catch (InterruptedException e) {
 308                     }
 309                 }
 310             } finally {
 311                 vpnIntfMap.remove(interfaceName);
 312             }
 313
 314             opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
 315             if (opVpnInterface != null) {
 316                 LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
 317                 return;
 318             }
 319             // VPNInterface got removed, proceed with Add
 320             updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 321             bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 322             processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 323         } else {
 324             // Interface is retained in the DPN, but its Link Up.
 325             // Advertise prefixes again for this interface to BGP
 326             advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
 327                     vpnInterface);
 328         }
 329     }
 330
 331
 332 //    private class UpdateDpnToVpnWorker implements Callable<List<ListenableFuture<Void>>> {
 333 //        BigInteger dpnId;
 334 //        String vpnName;
 335 //        String interfaceName;
 336 //        boolean addToDpn;
 337 //        int lPortTag;
 338 //
 339 //        public UpdateDpnToVpnWorker(BigInteger dpnId, String vpnName, String interfaceName,
 340 //                                    int lPortTag, boolean addToDpn) {
 341 //            this.dpnId= dpnId;
 342 //            this.vpnName = vpnName;
 343 //            this.interfaceName = interfaceName;
 344 //            this.lPortTag = lPortTag;
 345 //            this.addToDpn = addToDpn;
 346 //        }
 347 //
 348 //        @Override
 349 //        public List<ListenableFuture<Void>> call() throws Exception {
 350 //            // If another renderer(for eg : CSS) needs to be supported, check can be performed here
 351 //            // to call the respective helpers.
 352 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 353 //            updateDpnDbs(dpnId, vpnName, interfaceName, addToDpn, writeTxn);
 354 //            List<ListenableFuture<Void>> futures = new ArrayList<>();
 355 //            futures.add(writeTxn.submit());
 356 //            ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
 357 //            Futures.addCallback(listenableFuture,
 358 //                    new UpdateDpnToVpnCallback(dpnId, vpnName, interfaceName, lPortTag, addToDpn));
 359 //            return futures;
 360 //        }
 361 //    }
 362 //
 363 //
 364 //    /**
 365 //     * JobCallback class is used as a future callback for
 366 //     * main and rollback workers to handle success and failure.
 367 //     */
 368 //    private class UpdateDpnToVpnCallback implements FutureCallback<List<Void>> {
 369 //        BigInteger dpnId;
 370 //        String vpnName;
 371 //        String interfaceName;
 372 //        boolean addToDpn;
 373 //        int lPortTag;
 374 //
 375 //        public UpdateDpnToVpnCallback(BigInteger dpnId, String vpnName, String interfaceName,
 376 //                                      int lPortTag, boolean addToDpn) {
 377 //            this.dpnId= dpnId;
 378 //            this.vpnName = vpnName;
 379 //            this.interfaceName = interfaceName;
 380 //            this.lPortTag = lPortTag;
 381 //            this.addToDpn = addToDpn;
 382 //        }
 383 //
 384 //        /**
 385 //         * @param voids
 386 //         * This implies that all the future instances have returned success. -- TODO: Confirm this
 387 //         */
 388 //        @Override
 389 //        public void onSuccess(List<Void> voids) {
 390 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 391 //            bindService(dpnId, vpnName, interfaceName, lPortTag, writeTxn);
 392 //            processVpnInterfaceAdjacencies(dpnId, vpnName, interfaceName, writeTxn);
 393 //            writeTxn.submit();
 394 //        }
 395 //
 396 //        /**
 397 //         *
 398 //         * @param throwable
 399 //         * This method is used to handle failure callbacks.
 400 //         * If more retry needed, the retrycount is decremented and mainworker is executed again.
 401 //         * After retries completed, rollbackworker is executed.
 402 //         * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 403 //         */
 404 //
 405 //        @Override
 406 //        public void onFailure(Throwable throwable) {
 407 //            LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 408 //        }
 409 //    }
 410
 411
 412
 413
 414     private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
 415                                                  VpnInterface intf) {
 416         //Read NextHops
 417         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 418         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 419
 420         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 421         if (rd == null) {
 422             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 423                     intf.getName(), intf.getVpnInstanceName());
 424             return;
 425         } else {
 426             if (rd.equals(intf.getVpnInstanceName())) {
 427                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in " +
 428                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 429
 430                 return;
 431             }
 432         }
 433         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
 434                 intf.getVpnInstanceName(), rd);
 435
 436         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 437         if (nextHopIp == null){
 438             LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
 439             return;
 440         }
 441
 442         if (adjacencies.isPresent()) {
 443             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 444
 445             if (!nextHops.isEmpty()) {
 446                 LOG.trace("NextHops are " + nextHops);
 447                 for (Adjacency nextHop : nextHops) {
 448                     long label = nextHop.getLabel();
 449                     try {
 450                         LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 451                         bgpManager.advertisePrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
 452                         LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 453                     } catch(Exception e) {
 454                         LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
 455                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 456                     }
 457                 }
 458             }
 459         }
 460     }
 461
 462     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 463         //Read NextHops
 464         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 465         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 466
 467         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 468         if (rd == null) {
 469             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 470                     intf.getName(), intf.getVpnInstanceName());
 471             return;
 472         } else {
 473             if (rd.equals(intf.getVpnInstanceName())) {
 474                 LOG.info("withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in " +
 475                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 476                 return;
 477             }
 478         }
 479         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 480                 intf.getVpnInstanceName(), rd);
 481         if (adjacencies.isPresent()) {
 482             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 483
 484             if (!nextHops.isEmpty()) {
 485                 LOG.trace("NextHops are " + nextHops);
 486                 for (Adjacency nextHop : nextHops) {
 487                     try {
 488                         LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 489                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 490                         LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 491                     } catch(Exception e) {
 492                         LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
 493                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 494                     }
 495                 }
 496             }
 497         }
 498     }
 499
 500     public void updateVpnToDpnMapping(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 501         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 502         if (dpId == null) {
 503             dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, interfaceName);
 504         }
 505         if(!dpId.equals(BigInteger.ZERO)) {
 506             if(add)
 507                 createOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 508             else
 509                 removeOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 510         }
 511     }
 512
 513     private void bindService(BigInteger dpId, final String vpnInstanceName, final String vpnInterfaceName,
 514                              int lPortTag, WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
 515         final int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 516         final long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
 517
 518         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 519         dataStoreCoordinator.enqueueJob(vpnInterfaceName,
 520                 new Callable<List<ListenableFuture<Void>>>() {
 521                     @Override
 522                     public List<ListenableFuture<Void>> call() throws Exception {
 523                         WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 524                         int instructionKey = 0;
 525                         List<Instruction> instructions = new ArrayList<Instruction>();
 526
 527                         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(
 528                                 MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 529                         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 530
 531                         BoundServices
 532                                 serviceInfo =
 533                                 InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 534                                         NwConstants.L3VPN_SERVICE_INDEX, priority,
 535                                         NwConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 536                         writeTxn.put(LogicalDatastoreType.CONFIGURATION,
 537                                 InterfaceUtils.buildServiceId(vpnInterfaceName, NwConstants.L3VPN_SERVICE_INDEX), serviceInfo, true);
 538                         List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
 539                         futures.add(writeTxn.submit());
 540                         return futures;
 541                     }
 542                 });
 543         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
 544                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW, writeInvTxn);
 545         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
 546                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.ADD_FLOW, writeInvTxn);
 547
 548     }
 549
 550     private void processVpnInterfaceAdjacencies(BigInteger dpnId, String vpnName, String interfaceName,
 551                                                 WriteTransaction writeConfigTxn,
 552                                                 WriteTransaction writeOperTxn) {
 553         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 554         // Read NextHops
 555         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 556         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 557
 558         if (adjacencies.isPresent()) {
 559             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 560             List<Adjacency> value = new ArrayList<>();
 561
 562             // Get the rd of the vpn instance
 563             String rd = getRouteDistinguisher(vpnName);
 564
 565             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 566             if (nextHopIp == null){
 567                 LOG.error("NextHop for interface {} is null", interfaceName);
 568                 return;
 569             }
 570
 571             List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
 572
 573             LOG.trace("NextHops for interface {} are {}", interfaceName, nextHops);
 574             for (Adjacency nextHop : nextHops) {
 575                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 576                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 577                         VpnUtil.getNextHopLabelKey((rd == null) ? vpnName
 578                                 : rd, prefix));
 579                 List<String> adjNextHop = nextHop.getNextHopIpList();
 580                 value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIpList(
 581                         (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : Arrays.asList(nextHopIp))
 582                         .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 583
 584                 if (nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 585                     LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, interfaceName, vpnName);
 586                     writeOperTxn.merge(
 587                             LogicalDatastoreType.OPERATIONAL,
 588                             VpnUtil.getPrefixToInterfaceIdentifier(
 589                                     VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 590                             VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix), true);
 591                 } else {
 592                     //Extra route adjacency
 593                     LOG.trace("Adding prefix {} and nexthopList {} as extra-route for vpn", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName);
 594                     writeOperTxn.merge(
 595                             LogicalDatastoreType.OPERATIONAL,
 596                             VpnUtil.getVpnToExtrarouteIdentifier(
 597                                     (rd != null) ? rd : vpnName, nextHop.getIpAddress()),
 598                             VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()), true);
 599                 }
 600             }
 601
 602             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 603
 604             VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 605             InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 606             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, true);
 607             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 608
 609             for (Adjacency nextHop : aug.getAdjacency()) {
 610                 long label = nextHop.getLabel();
 611                 if (rd != null) {
 612                     addToLabelMapper(label, dpnId, nextHop.getIpAddress(), Arrays.asList(nextHopIp), vpnId,
 613                             interfaceName, null,false, rd, writeOperTxn);
 614                     addPrefixToBGP(rd, nextHop.getIpAddress(), nextHopIp, label, writeConfigTxn);
 615                     //TODO: ERT - check for VPNs importing my route
 616                     for (VpnInstance vpn : vpnsToImportRoute) {
 617                         String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
 618                         if (vpnRd != null) {
 619                             LOG.debug("Exporting route with rd {} prefix {} nexthop {} label {} to VPN {}", vpnRd, nextHop.getIpAddress(), nextHopIp, label, vpn);
 620                             fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), Arrays.asList(nextHopIp), (int) label,
 621                                     RouteOrigin.SELF_IMPORTED, writeConfigTxn);
 622                         }
 623                     }
 624                 } else {
 625                     // ### add FIB route directly
 626                     fibManager.addOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), Arrays.asList(nextHopIp),
 627                             (int) label, RouteOrigin.STATIC, writeConfigTxn);
 628                 }
 629             }
 630         }
 631     }
 632
 633     private List<VpnInstance> getVpnsImportingMyRoute(final String vpnName) {
 634         List<VpnInstance> vpnsToImportRoute = new ArrayList<>();
 635
 636         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 637                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 638         Optional<VpnInstance> optVpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 639         final VpnInstance vpnInstance;
 640         if (optVpnInstance.isPresent()) {
 641             vpnInstance = optVpnInstance.get();
 642         } else {
 643             LOG.debug("Could not retrieve vpn instance {} to check for vpns importing the routes", vpnName);
 644             return vpnsToImportRoute;
 645         }
 646
 647         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 648             @Override
 649             public boolean apply(VpnInstance input) {
 650                 return !input.getVpnInstanceName().equals(vpnName);
 651             }
 652         };
 653
 654         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 655             @Override
 656             public boolean apply(VpnInstance input) {
 657                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ExportExtcommunity),
 658                         getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 659                 return Iterators.size(commonRTs.iterator()) > 0;
 660             }
 661         };
 662
 663         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 664             @Override
 665             public String apply(VpnInstance vpnInstance) {
 666                 //return vpnInstance.getVpnInstanceName();
 667                 return vpnInstance.getIpv4Family().getRouteDistinguisher();
 668             }
 669         };
 670
 671         vpnsToImportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(dataBroker)).
 672                 filter(excludeVpn).
 673                 filter(matchRTs).toList();
 674         return vpnsToImportRoute;
 675     }
 676
 677     private List<VpnInstance> getVpnsExportingMyRoute(final String vpnName) {
 678         List<VpnInstance> vpnsToExportRoute = new ArrayList<>();
 679
 680         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 681                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 682         Optional<VpnInstance> optVpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 683         final VpnInstance vpnInstance;
 684         if (optVpnInstance.isPresent()) {
 685             vpnInstance = optVpnInstance.get();
 686         } else {
 687             LOG.debug("Could not retrieve vpn instance {} to check for vpns exporting the routes", vpnName);
 688             return vpnsToExportRoute;
 689         }
 690
 691         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 692             @Override
 693             public boolean apply(VpnInstance input) {
 694                 return !input.getVpnInstanceName().equals(vpnName);
 695             }
 696         };
 697
 698         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 699             @Override
 700             public boolean apply(VpnInstance input) {
 701                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ImportExtcommunity),
 702                         getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
 703                 return Iterators.size(commonRTs.iterator()) > 0;
 704             }
 705         };
 706
 707         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 708             @Override
 709             public String apply(VpnInstance vpnInstance) {
 710                 return vpnInstance.getVpnInstanceName();
 711             }
 712         };
 713
 714         vpnsToExportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(dataBroker)).
 715                 filter(excludeVpn).
 716                 filter(matchRTs).toList();
 717         return vpnsToExportRoute;
 718     }
 719
 720     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
 721         final Predicate<T> inPredicate = Predicates.<T>in(collection2);
 722         return new Iterable<T>() {
 723             @Override
 724             public Iterator<T> iterator() {
 725                 return Iterators.filter(collection1.iterator(), inPredicate);
 726             }
 727         };
 728     }
 729
 730     private List<String> getRts(VpnInstance vpnInstance, VpnTarget.VrfRTType rtType) {
 731         String name = vpnInstance.getVpnInstanceName();
 732         List<String> rts = new ArrayList<>();
 733         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 734         if (vpnConfig == null) {
 735             LOG.trace("vpn config is not available for {}", name);
 736             return rts;
 737         }
 738         VpnTargets targets = vpnConfig.getVpnTargets();
 739         if (targets == null) {
 740             LOG.trace("vpn targets not available for {}", name);
 741             return rts;
 742         }
 743         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 744         if (vpnTargets == null) {
 745             LOG.trace("vpnTarget values not available for {}", name);
 746             return rts;
 747         }
 748         for (VpnTarget target : vpnTargets) {
 749             //TODO: Check for RT type is Both
 750             if(target.getVrfRTType().equals(rtType) ||
 751                     target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
 752                 String rtValue = target.getVrfRTValue();
 753                 rts.add(rtValue);
 754             }
 755         }
 756         return rts;
 757     }
 758
 759     private List<String> getExportRts(VpnInstance vpnInstance) {
 760         List<String> exportRts = new ArrayList<>();
 761         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 762         VpnTargets targets = vpnConfig.getVpnTargets();
 763         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 764         for (VpnTarget target : vpnTargets) {
 765             if (target.getVrfRTType().equals(VpnTarget.VrfRTType.ExportExtcommunity)) {
 766                 String rtValue = target.getVrfRTValue();
 767                 exportRts.add(rtValue);
 768             }
 769         }
 770         return exportRts;
 771     }
 772
 773     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 774                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow,
 775                              WriteTransaction writeConfigTxn){
 776         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 777         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, MetaDataUtil.getVpnIdMetadata(vpnId));
 778         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 779                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 780
 781         // Matching Arp reply flows
 782         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 783         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 784                 metadata, metadataMask }));
 785
 786         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 787
 788         // Instruction to punt to controller
 789         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 790         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 791         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 792         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
 793                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
 794
 795         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 796
 797         // Install the flow entry in L3_INTERFACE_TABLE
 798         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 799                 NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 800         FlowEntity flowEntity;
 801         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 802                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 803                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 804
 805         Flow flow = flowEntity.getFlowBuilder().build();
 806         String flowId = flowEntity.getFlowId();
 807         FlowKey flowKey = new FlowKey( new FlowId(flowId));
 808         Node nodeDpn = buildDpnNode(dpId);
 809
 810         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 811                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 812                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 813
 814         if (writeConfigTxn != null) {
 815             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 816                 LOG.debug("Creating ARP Flow for interface {}", vpnInterfaceName);
 817                 writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
 818             } else {
 819                 LOG.debug("Deleting ARP Flow for interface {}", vpnInterfaceName);
 820                 writeConfigTxn.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 821             }
 822         } else {
 823             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 824                 LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 825                 mdsalManager.installFlow(flowEntity);
 826             } else {
 827                 LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 828                 mdsalManager.removeFlow(flowEntity);
 829             }
 830         }
 831     }
 832
 833     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
 834     private Node buildDpnNode(BigInteger dpnId) {
 835         NodeId nodeId = new NodeId("openflow:" + dpnId);
 836         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
 837
 838         return nodeDpn;
 839     }
 840
 841     private String getRouteDistinguisher(String vpnName) {
 842         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 843                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 844         Optional<VpnInstance> vpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 845         String rd = null;
 846         if(vpnInstance.isPresent()) {
 847             VpnInstance instance = vpnInstance.get();
 848             VpnAfConfig config = instance.getIpv4Family();
 849             rd = config.getRouteDistinguisher();
 850         }
 851         return rd;
 852     }
 853
 854     /**
 855      * JobCallback class is used as a future callback for
 856      * main and rollback workers to handle success and failure.
 857      */
 858     private class DpnEnterExitVpnWorker implements FutureCallback<List<Void>> {
 859         BigInteger dpnId;
 860         String vpnName;
 861         String rd;
 862         boolean entered;
 863
 864         public DpnEnterExitVpnWorker(BigInteger dpnId, String vpnName, String rd, boolean entered) {
 865             this.entered = entered;
 866             this.dpnId = dpnId;
 867             this.vpnName = vpnName;
 868             this.rd = rd;
 869         }
 870
 871         /**
 872          * @param voids
 873          * This implies that all the future instances have returned success. -- TODO: Confirm this
 874          */
 875         @Override
 876         public void onSuccess(List<Void> voids) {
 877             if (entered) {
 878                 publishAddNotification(dpnId, vpnName, rd);
 879             } else {
 880                 publishRemoveNotification(dpnId, vpnName, rd);
 881             }
 882         }
 883
 884         /**
 885          *
 886          * @param throwable
 887          * This method is used to handle failure callbacks.
 888          * If more retry needed, the retrycount is decremented and mainworker is executed again.
 889          * After retries completed, rollbackworker is executed.
 890          * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 891          */
 892         @Override
 893         public void onFailure(Throwable throwable) {
 894             LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 895         }
 896     }
 897
 898     private void createOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 899         String routeDistinguisher = getRouteDistinguisher(vpnName);
 900         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 901         Boolean newDpnOnVpn = Boolean.FALSE;
 902
 903         synchronized (vpnName.intern()) {
 904             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 905             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 906             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 907             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data
 908                     .entry.vpn.to.dpn.list.VpnInterfaces
 909                     vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 910
 911             if (dpnInVpn.isPresent()) {
 912                 VpnToDpnList vpnToDpnList = dpnInVpn.get();
 913                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 914                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = vpnToDpnList.getVpnInterfaces();
 915                 if (vpnInterfaces == null) {
 916                     vpnInterfaces = new ArrayList<>();
 917                 }
 918                 vpnInterfaces.add(vpnInterface);
 919                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder(vpnToDpnList);
 920                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 921
 922                 if (writeTxn != null) {
 923                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 924                 } else {
 925                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 926                 }
 927                 /* If earlier state was inactive, it is considered new DPN coming back to the
 928                  * same VPN
 929                  */
 930                 if (vpnToDpnList.getDpnState() == VpnToDpnList.DpnState.Inactive) {
 931                     newDpnOnVpn = Boolean.TRUE;
 932                 }
 933             } else {
 934                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 935                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = new ArrayList<>();
 936                 vpnInterfaces.add(vpnInterface);
 937                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder().setDpnId(dpnId);
 938                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 939
 940                 if (writeTxn != null) {
 941                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 942                 } else {
 943                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 944                 }
 945                 newDpnOnVpn = Boolean.TRUE;
 946             }
 947             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
 948             try {
 949                 futures.get();
 950             } catch (InterruptedException | ExecutionException e) {
 951                 LOG.error("Error adding to dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
 952                 throw new RuntimeException(e.getMessage());
 953             }
 954         }
 955         /*
 956          * Informing the Fib only after writeTxn is submitted successfuly.
 957          */
 958         if (newDpnOnVpn) {
 959             LOG.debug("Sending populateFib event for new dpn {} in VPN {}", dpnId, vpnName);
 960             fibManager.populateFibOnNewDpn(dpnId, vpnId, rd, new DpnEnterExitVpnWorker(dpnId, vpnName, rd, true /* entered */));
 961         }
 962     }
 963
 964     private void removeOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 965         Boolean lastDpnOnVpn = Boolean.FALSE;
 966         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
 967         synchronized (vpnName.intern()) {
 968             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 969             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 970             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 971             if (dpnInVpn.isPresent()) {
 972                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 973                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 974                 org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 975                         currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 976
 977                 if (vpnInterfaces.remove(currVpnInterface)) {
 978                     if (vpnInterfaces.isEmpty()) {
 979                         List<IpAddresses> ipAddresses = dpnInVpn.get().getIpAddresses();
 980                         if (ipAddresses == null || ipAddresses.isEmpty()) {
 981                             VpnToDpnListBuilder dpnInVpnBuilder =
 982                                     new VpnToDpnListBuilder(dpnInVpn.get())
 983                                             .setDpnState(VpnToDpnList.DpnState.Inactive)
 984                                             .setVpnInterfaces(null);
 985                             if (writeTxn != null) {
 986                                 writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build(), true);
 987                             } else {
 988                                 VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build());
 989                             }
 990                             lastDpnOnVpn = Boolean.TRUE;
 991                         } else {
 992                             LOG.warn("vpn interfaces are empty but ip addresses are present for the vpn {} in dpn {}", vpnName, dpnId);
 993                         }
 994                     } else {
 995                         if (writeTxn != null) {
 996                             writeTxn.delete(LogicalDatastoreType.OPERATIONAL, id.child(
 997                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 998                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 999                                     new VpnInterfacesKey(intfName)));
1000                         } else {
1001                             VpnUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1002                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1003                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1004                                     new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
1005                         }
1006                     }
1007                 }
1008             }
1009             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
1010             try {
1011                 futures.get();
1012             } catch (InterruptedException | ExecutionException e) {
1013                 LOG.error("Error removing from dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
1014                 throw new RuntimeException(e.getMessage());
1015             }
1016         }
1017         if (lastDpnOnVpn) {
1018             LOG.debug("Sending cleanup event for dpn {} in VPN {}", dpnId, vpnName);
1019             fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd, new DpnEnterExitVpnWorker(dpnId, vpnName, rd, false /* exited */));
1020         }
1021     }
1022
1023     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1024         List<VpnInstance> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1025         for (VpnInstance vpn : vpnsToExportRoute) {
1026             String rd = vpn.getIpv4Family().getRouteDistinguisher();
1027             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getIpv4Family().getRouteDistinguisher());
1028             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1029             if (vrfEntries != null) {
1030                 for (VrfEntry vrfEntry : vrfEntries) {
1031                     try {
1032                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1033                             continue;
1034                         }
1035                         String prefix = vrfEntry.getDestPrefix();
1036                         long label = vrfEntry.getLabel();
1037                         List<String> nextHops = vrfEntry.getNextHopAddressList();
1038                         SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1039                         for (String nh : nextHops) {
1040                             if (route != null) {
1041                                 LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1042                                 importSubnetRouteForNewVpn(rd, prefix, nh, (int)label, route, writeConfigTxn);
1043                             } else {
1044                                 LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1045                                 fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, prefix, Arrays.asList(nh), (int)label,
1046                                         RouteOrigin.SELF_IMPORTED, writeConfigTxn);
1047                             }
1048                         }
1049                     } catch (Exception e) {
1050                         LOG.error("Exception occurred while importing route with prefix {} label {} nexthop {} from vpn {} to vpn {}", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpn.getVpnInstanceName(), vpnName);
1051                     }
1052                 }
1053                 writeConfigTxn.submit();
1054             } else {
1055                 LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getIpv4Family().getRouteDistinguisher());
1056             }
1057         }
1058     }
1059
1060     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label, WriteTransaction writeConfigTxn) {
1061         try {
1062             LOG.info("ADD: Adding Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1063             fibManager.addOrUpdateFibEntry(dataBroker, rd, prefix, Arrays.asList(nextHopIp), (int)label, RouteOrigin.STATIC, writeConfigTxn);
1064             bgpManager.advertisePrefix(rd, prefix, Arrays.asList(nextHopIp), (int)label);
1065             LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1066         } catch(Exception e) {
1067             LOG.error("Add prefix failed", e);
1068         }
1069     }
1070
1071     @Override
1072     public void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1073         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
1074         LOG.info("VPN Interface remove event - intfName {}" ,vpnInterface.getName());
1075         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1076         final String interfaceName = key.getName();
1077
1078         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1079         final Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1080         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
1081                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1082         if (optVpnInterface.isPresent()){
1083             BigInteger dpnId = BigInteger.ZERO;
1084             Boolean dpnIdRetrieved = Boolean.FALSE;
1085             if(interfaceState != null){
1086                 try{
1087                     dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1088                     dpnIdRetrieved = Boolean.TRUE;
1089                 }catch (Exception e){
1090                     LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", interfaceName, e);
1091                 }
1092             } else {
1093                 LOG.error("Unable to retrieve interfaceState for interface {} , quitting ", interfaceName);
1094                 return;
1095             }
1096             final VpnInterface vpnOpInterface = optVpnInterface.get();
1097             if(dpnIdRetrieved == Boolean.FALSE){
1098                 LOG.info("dpnId for {} has not been retrieved yet. Fetching from vpn interface operational DS", interfaceName);
1099                 dpnId = vpnOpInterface.getDpnId();
1100             }
1101             final int ifIndex = interfaceState.getIfIndex();
1102             final BigInteger dpId = dpnId;
1103             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1104             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1105                     new Callable<List<ListenableFuture<Void>>>() {
1106                         @Override
1107                         public List<ListenableFuture<Void>> call() throws Exception {
1108                             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1109                             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1110                             WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
1111                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, false, true, writeConfigTxn, writeOperTxn, writeInvTxn);
1112                             List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
1113                             futures.add(writeOperTxn.submit());
1114                             futures.add(writeConfigTxn.submit());
1115                             futures.add(writeInvTxn.submit());
1116                             return futures;
1117                         }
1118                     });
1119
1120         }else{
1121             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
1122         }
1123     }
1124
1125     protected void processVpnInterfaceDown(BigInteger dpId,
1126                                            String interfaceName,
1127                                            int lPortTag,
1128                                            boolean isInterfaceStateDown,
1129                                            boolean isConfigRemoval,
1130                                            WriteTransaction writeConfigTxn,
1131                                            WriteTransaction writeOperTxn,
1132                                            WriteTransaction writeInvTxn) {
1133         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1134         if (!isInterfaceStateDown) {
1135             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1136             if(vpnInterface == null){
1137                 LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
1138                 return;
1139             }else{
1140                 final String vpnName = vpnInterface.getVpnInstanceName();
1141                 if(!vpnInterface.isScheduledForRemove()){
1142                     VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE, writeOperTxn);
1143                     removeAdjacenciesFromVpn(dpId, interfaceName, vpnInterface.getVpnInstanceName(), writeConfigTxn);
1144                     LOG.info("Unbinding vpn service from interface {} ", interfaceName);
1145                     unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown, isConfigRemoval, writeConfigTxn, writeInvTxn);
1146                 }else{
1147                     LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ", interfaceName);
1148                     return;
1149                 }
1150             }
1151         } else {
1152             // Interface is retained in the DPN, but its Link Down.
1153             // Only withdraw the prefixes for this interface from BGP
1154             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1155             if(vpnInterface == null){
1156                 LOG.info("Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in operational data store", interfaceName);
1157                 return;
1158             }else {
1159                 withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
1160             }
1161         }
1162     }
1163
1164     private void waitForFibToRemoveVpnPrefix(String interfaceName) {
1165         // FIB didn't get a chance yet to clean up this VPNInterface
1166         // Let us give it a chance here !
1167         LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
1168         try {
1169             Runnable notifyTask = new VpnNotifyTask();
1170             vpnIntfMap.put(interfaceName, notifyTask);
1171             synchronized (notifyTask) {
1172                 try {
1173                     notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
1174                 } catch (InterruptedException e) {
1175                 }
1176             }
1177         } finally {
1178             vpnIntfMap.remove(interfaceName);
1179         }
1180     }
1181
1182     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final String interfaceName, final String vpnName,
1183                                           WriteTransaction writeConfigTxn) {
1184         //Read NextHops
1185         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1186         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1187         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1188
1189         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1190         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", interfaceName,
1191                 vpnName, rd);
1192         if (adjacencies.isPresent()) {
1193             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1194
1195             if (!nextHops.isEmpty()) {
1196                 LOG.trace("NextHops are " + nextHops);
1197                 for (Adjacency nextHop : nextHops) {
1198                     List<String> nhList = new ArrayList<String>();
1199                     if (nextHop.getMacAddress() == null || nextHop.getMacAddress().isEmpty()) {
1200                         // This is either an extra-route (or) a learned IP via subnet-route
1201                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1202                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1203                             LOG.error("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
1204                                     rd, nextHop.getIpAddress());
1205                             continue;
1206                         }
1207                         nhList = Arrays.asList(nextHopIp);
1208                     } else {
1209                         // This is a primary adjacency
1210                         nhList = nextHop.getNextHopIpList();
1211                     }
1212                     if (rd.equals(vpnName)) {
1213                         //this is an internal vpn - the rd is assigned to the vpn instance name;
1214                         //remove from FIB directly
1215                         for(String nh : nhList) {
1216                             fibManager.removeOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), nh, writeConfigTxn);
1217                         }
1218                     } else {
1219                         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1220                         for (String nh : nhList) {
1221                             //IRT: remove routes from other vpns importing it
1222                             removePrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1223                             for (VpnInstance vpn : vpnsToImportRoute) {
1224                                 String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
1225                                 if (vpnRd != null) {
1226                                     LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd, nextHop.getIpAddress(), vpn.getVpnInstanceName());
1227                                     fibManager.removeOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), nh, writeConfigTxn);
1228                                 }
1229                             }
1230                         }
1231                     }
1232                     String ip = nextHop.getIpAddress().split("/")[0];
1233                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
1234                             vpnName, ip);
1235                     if (vpnPortipToPort != null && !vpnPortipToPort.isConfig()) {
1236                         LOG.trace("VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
1237                                 vpnPortipToPort.getPortName(),ip);
1238                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip);
1239                     }
1240                 }
1241             }
1242         }
1243     }
1244
1245
1246     private void unbindService(BigInteger dpId, String vpnInstanceName, final String vpnInterfaceName,
1247                                int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval,
1248                                WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
1249         short l3vpn_service_index = ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX);
1250         if (!isInterfaceStateDown && isConfigRemoval) {
1251             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1252             dataStoreCoordinator.enqueueJob(vpnInterfaceName,
1253                     new Callable<List<ListenableFuture<Void>>>() {
1254                         @Override
1255                         public List<ListenableFuture<Void>> call() throws Exception {
1256                             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
1257                             writeTxn.delete(LogicalDatastoreType.CONFIGURATION,
1258                                     InterfaceUtils.buildServiceId(vpnInterfaceName,
1259                                             NwConstants.L3VPN_SERVICE_INDEX));
1260
1261                             List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
1262                             futures.add(writeTxn.submit());
1263                             return futures;
1264                         }
1265                     });
1266         }
1267         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
1268         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
1269                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW, writeInvTxn);
1270         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
1271                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.DEL_FLOW, writeInvTxn);
1272     }
1273
1274
1275     private void removePrefixFromBGP(String rd, String prefix, String nextHop, WriteTransaction writeConfigTxn) {
1276         try {
1277             LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
1278             fibManager.removeOrUpdateFibEntry(dataBroker, rd, prefix, nextHop, writeConfigTxn);
1279             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1280             LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
1281         } catch(Exception e) {
1282             LOG.error("Delete prefix failed", e);
1283         }
1284     }
1285
1286     @Override
1287     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
1288         LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
1289         LOG.info("VPN Interface update event - intfName {}" ,update.getName());
1290         String oldVpnName = original.getVpnInstanceName();
1291         String newVpnName = update.getVpnInstanceName();
1292         BigInteger dpnId = update.getDpnId();
1293         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
1294         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
1295         if (oldAdjs == null) {
1296             oldAdjs = new ArrayList<>();
1297         }
1298         if (newAdjs == null) {
1299             newAdjs = new ArrayList<>();
1300         }
1301         //handles switching between <internal VPN - external VPN>
1302         if (!oldVpnName.equals(newVpnName)) {
1303             remove(identifier, original);
1304             waitForFibToRemoveVpnPrefix(update.getName());
1305             add(identifier, update);
1306         }
1307         //handle both addition and removal of adjacencies
1308         //currently, new adjacency may be an extra route
1309         if (!oldAdjs.equals(newAdjs)) {
1310             for (Adjacency adj : newAdjs) {
1311                 if (oldAdjs.contains(adj)) {
1312                     oldAdjs.remove(adj);
1313                 } else {
1314                     // add new adjacency - right now only extra route will hit this path
1315                     addNewAdjToVpnInterface(identifier, adj, dpnId);
1316                 }
1317             }
1318             for (Adjacency adj : oldAdjs) {
1319                 delAdjFromVpnInterface(identifier, adj, dpnId);
1320             }
1321         }
1322     }
1323
1324     public void processArpRequest(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715
1325                                           .IpAddress srcIP, PhysAddress srcMac, org.opendaylight.yang.gen.v1.urn.ietf
1326             .params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress targetIP, PhysAddress targetMac, String srcInterface){
1327         //Build ARP response with ARP requests TargetIp TargetMac as the Arp Response SrcIp and SrcMac
1328         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
1329                 .setDstIpaddress(srcIP).setDstMacaddress(srcMac).setSrcIpaddress(targetIP).setSrcMacaddress(targetMac).build();
1330         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
1331         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
1332         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
1333             @Override
1334             public void onFailure(Throwable error) {
1335                 LOG.error("Error - {}", msgFormat, error);
1336             }
1337
1338             @Override
1339             public void onSuccess(RpcResult<Void> result) {
1340                 if(!result.isSuccessful()) {
1341                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
1342                 } else {
1343                     LOG.debug("Successful RPC Result - {}", msgFormat);
1344                 }
1345             }
1346         });
1347     }
1348
1349     private String getErrorText(Collection<RpcError> errors) {
1350         StringBuilder errorText = new StringBuilder();
1351         for(RpcError error : errors) {
1352             errorText.append(",").append(error.getErrorType()).append("-")
1353                     .append(error.getMessage());
1354         }
1355         return errorText.toString();
1356     }
1357
1358     private void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1359                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd,
1360                                   WriteTransaction writeOperTxn) {
1361         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1362         Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
1363         Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
1364         Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
1365         if (!isSubnetRoute) {
1366             // NextHop must be present for non-subnetroute entries
1367             Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
1368         }
1369         LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
1370         if (dpnId != null) {
1371             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1372                     .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
1373             LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1374             lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(vpnId)
1375                     .setIsSubnetRoute(isSubnetRoute);
1376             if (elanTag != null) {
1377                 lriBuilder.setElanTag(elanTag);
1378             }
1379             if (vpnInterfaceName != null) {
1380                 lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1381             }
1382             lriBuilder.setParentVpnRd(rd);
1383             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
1384             if (vpnInstanceOpDataEntry != null) {
1385                 List<String> vpnInstanceNames = Arrays.asList(vpnInstanceOpDataEntry.getVpnInstanceName());
1386                 lriBuilder.setVpnInstanceList(vpnInstanceNames);
1387             }
1388             LabelRouteInfo lri = lriBuilder.build();
1389             LOG.trace("Adding route info to label map: {}", lri);
1390             if (writeOperTxn != null) {
1391                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, lriIid, lri, true);
1392             } else {
1393                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
1394             }
1395         } else {
1396             LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
1397         }
1398     }
1399
1400     public synchronized void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
1401                                                         long elantag, BigInteger dpnId, WriteTransaction writeTxn) {
1402         SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
1403         RouteOrigin origin = RouteOrigin.STATIC; // Only case when a route is considered as directly connected
1404         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1405                 .setLabel((long)label).setOrigin(origin.getValue())
1406                 .addAugmentation(SubnetRoute.class, route).build();
1407
1408         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
1409
1410         //TODO: What should be parentVpnId? Get it from RD?
1411         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1412         addToLabelMapper((long)label, dpnId, prefix, Arrays.asList(nextHop), vpnId, null, elantag, true, rd, null);
1413         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1414
1415         InstanceIdentifierBuilder<VrfTables> idBuilder =
1416                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1417         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1418
1419         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1420                 setVrfEntry(vrfEntryList).build();
1421
1422         if (writeTxn != null) {
1423             writeTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1424         } else {
1425             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1426         }
1427
1428         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1429         if (vpnsToImportRoute.size() > 0) {
1430             VrfEntry importingVrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1431                     .setLabel((long) label).setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
1432                     .addAugmentation(SubnetRoute.class, route).build();
1433             List<VrfEntry> importingVrfEntryList = Arrays.asList(importingVrfEntry);
1434             for (VpnInstance vpnInstance : vpnsToImportRoute) {
1435                 LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop, label, vpnInstance.getVpnInstanceName());
1436                 String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1437                 InstanceIdentifier<VrfTables> importingVrfTableId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(importingRd)).build();
1438                 VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(importingVrfEntryList).build();
1439                 if (writeTxn != null) {
1440                     writeTxn.merge(LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable, true);
1441                 } else {
1442                     VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable);
1443                 }
1444             }
1445         }
1446     }
1447
1448     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1449                                                         SubnetRoute route, WriteTransaction writeConfigTxn) {
1450
1451         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1452         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1453                 .setLabel((long)label).setOrigin(origin.getValue())
1454                 .addAugmentation(SubnetRoute.class, route).build();
1455         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, route.getElantag());
1456         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1457         InstanceIdentifierBuilder<VrfTables> idBuilder =
1458                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1459         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1460         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1461                 setVrfEntry(vrfEntryList).build();
1462         if (writeConfigTxn != null) {
1463             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1464         } else {
1465             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1466         }
1467     }
1468
1469     public synchronized void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName){
1470         fibManager.removeFibEntry(dataBroker, rd, prefix, null);
1471         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1472         for (VpnInstance vpnInstance : vpnsToImportRoute) {
1473             String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1474             LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix, vpnInstance.getVpnInstanceName());
1475             fibManager.removeFibEntry(dataBroker, importingRd, prefix, null);
1476         }
1477     }
1478
1479     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1480
1481         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1482
1483         if (optVpnInterface.isPresent()) {
1484             VpnInterface currVpnIntf = optVpnInterface.get();
1485             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1486             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1487             rd = (rd != null) ? rd : currVpnIntf.getVpnInstanceName();
1488             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1489             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, adjPath);
1490             long label =
1491                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1492                             VpnUtil.getNextHopLabelKey(rd, prefix));
1493
1494             List<Adjacency> adjacencies;
1495             if (optAdjacencies.isPresent()) {
1496                 adjacencies = optAdjacencies.get().getAdjacency();
1497             } else {
1498                 //This code will not be hit since VM adjacency will always be there
1499                 adjacencies = new ArrayList<>();
1500             }
1501
1502             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIpList(adj.getNextHopIpList())
1503                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
1504
1505             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1506             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove());
1507
1508             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1509             for (String nh : adj.getNextHopIpList()) {
1510                 addExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(), (int) label,
1511                         currVpnIntf.getName());
1512             }
1513         }
1514     }
1515
1516     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1517         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1518
1519         if (optVpnInterface.isPresent()) {
1520             VpnInterface currVpnIntf = optVpnInterface.get();
1521
1522             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1523             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1524             if (optAdjacencies.isPresent()) {
1525                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1526
1527                 if (!adjacencies.isEmpty()) {
1528                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1529                     rd = (rd != null) ? rd :currVpnIntf.getVpnInstanceName();
1530                     LOG.trace("Adjacencies are " + adjacencies);
1531                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1532                     while (adjIt.hasNext()) {
1533                         Adjacency adjElem = adjIt.next();
1534                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1535                             adjIt.remove();
1536
1537                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1538                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1539                                     currVpnIntf.getVpnInstanceName(),
1540                                     aug, dpnId, currVpnIntf.isScheduledForRemove());
1541
1542                             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1543
1544                             for (String nh : adj.getNextHopIpList()) {
1545                                 delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
1546                                         currVpnIntf.getName());
1547                             }
1548                             break;
1549                         }
1550
1551                     }
1552                 }
1553             }
1554         }
1555
1556     }
1557
1558     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label,
1559                                  String intfName) {
1560
1561         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1562         VpnUtil.syncUpdate(
1563                 dataBroker,
1564                 LogicalDatastoreType.OPERATIONAL,
1565                 VpnUtil.getVpnToExtrarouteIdentifier( (rd != null) ? rd : routerID, destination),
1566                 VpnUtil.getVpnToExtraroute(destination, Arrays.asList(nextHop)));
1567
1568         BigInteger dpnId = null;
1569         if (intfName != null && !intfName.isEmpty()) {
1570             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1571             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1572             if (nextHopIp == null || nextHopIp.isEmpty()) {
1573                 LOG.error("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1574                         intfName, destination);
1575                 return;
1576             }
1577             nextHop = nextHopIp;
1578         }
1579         List<String> nextHopIpList = Arrays.asList(nextHop);
1580         if (rd != null) {
1581             /* Label mapper is required only for BGP VPN and not for Internal VPN */
1582             addToLabelMapper((long) label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(dataBroker, routerID),
1583                     intfName, null, false, rd, null);
1584         }
1585
1586         // TODO (eperefr): This is a limitation to be stated in docs. When configuring static route to go to
1587         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1588         // first place.
1589         Optional<InterVpnLink> optInterVpnLink = InterVpnLinkUtil.getInterVpnLinkByEndpointIp(dataBroker, nextHop);
1590         if ( optInterVpnLink.isPresent() ) {
1591             InterVpnLink interVpnLink = optInterVpnLink.get();
1592             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1593             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1594             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1595             // This is like leaking one of the Vpn2 routes towards Vpn1
1596             boolean nexthopIsVpn2 = ( interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nextHop) );
1597             String srcVpnUuid = (nexthopIsVpn2) ? interVpnLink.getSecondEndpoint().getVpnUuid().getValue()
1598                     : interVpnLink.getFirstEndpoint().getVpnUuid().getValue();
1599             String dstVpnUuid = (nexthopIsVpn2) ? interVpnLink.getFirstEndpoint().getVpnUuid().getValue()
1600                     : interVpnLink.getSecondEndpoint().getVpnUuid().getValue();
1601             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
1602             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1603                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1604             InterVpnLinkUtil.leakRoute(dataBroker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel);
1605         } else {
1606             if (rd != null) {
1607                 addPrefixToBGP(rd, destination, nextHop, label, null);
1608             } else {
1609                 // ### add FIB route directly
1610                 fibManager.addOrUpdateFibEntry(dataBroker, routerID, destination, Arrays.asList(nextHop), label, RouteOrigin.STATIC, null);
1611             }
1612         }
1613     }
1614
1615     protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName) {
1616         if (intfName != null && !intfName.isEmpty()) {
1617             BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1618             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1619             if (nextHopIp == null || nextHopIp.isEmpty()) {
1620                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1621                         intfName, destination);
1622             }
1623             nextHop = nextHopIp;
1624         }
1625
1626         if (rd != null) {
1627             removePrefixFromBGP(rd, destination, nextHop, null);
1628         } else {
1629             // ### add FIB route directly
1630             fibManager.removeOrUpdateFibEntry(dataBroker, routerID, destination, nextHop, null);
1631         }
1632     }
1633
1634     void publishAddNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1635         LOG.debug("Sending notification for add dpn {} in vpn {} event ", dpnId, vpnName);
1636         AddEventData data = new AddEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1637         AddDpnEvent event = new AddDpnEventBuilder().setAddEventData(data).build();
1638         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1639         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1640             @Override
1641             public void onFailure(Throwable error) {
1642                 LOG.warn("Error in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName, error);
1643             }
1644
1645             @Override
1646             public void onSuccess(Object arg) {
1647                 LOG.trace("Successful in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName);
1648             }
1649         });
1650     }
1651
1652     void publishRemoveNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1653         LOG.debug("Sending notification for remove dpn {} in vpn {} event ", dpnId, vpnName);
1654         RemoveEventData data = new RemoveEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1655         RemoveDpnEvent event = new RemoveDpnEventBuilder().setRemoveEventData(data).build();
1656         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1657         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1658             @Override
1659             public void onFailure(Throwable error) {
1660                 LOG.warn("Error in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName, error);
1661             }
1662
1663             @Override
1664             public void onSuccess(Object arg) {
1665                 LOG.trace("Successful in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName);
1666             }
1667         });
1668     }
1669
1670     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1671         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1672                 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1673                 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1674     }
1675
1676     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1677         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1678                 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1679     }
1680
1681     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1682         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1683         if(dpId.equals(BigInteger.ZERO)) {
1684             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
1685             return;
1686         }
1687         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1688
1689         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1690                 .OPERATIONAL, routerDpnListIdentifier);
1691         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1692         if (optionalRouterDpnList.isPresent()) {
1693             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1694                     RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1695         } else {
1696             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1697             builder.setRouterId(routerName);
1698             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1699             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
1700             routerInterfaces.add(routerInterface);
1701             builder.setDpnVpninterfacesList(Arrays.asList(dpnVpnList.build()));
1702             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1703                     getRouterId(routerName),
1704                     builder.build(), true);
1705         }
1706     }
1707
1708     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1709         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1710         if(dpId.equals(BigInteger.ZERO)) {
1711             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1712             return;
1713         }
1714         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1715         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1716                 .OPERATIONAL, routerDpnListIdentifier);
1717         if (optionalRouterDpnList.isPresent()) {
1718             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1719             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1720
1721             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1722                 if (routerInterfaces.isEmpty()) {
1723                     if (writeOperTxn != null) {
1724                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1725                     } else {
1726                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1727                     }
1728                 } else {
1729                     if (writeOperTxn != null) {
1730                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1731                                 RouterInterfaces.class,
1732                                 new RouterInterfacesKey(vpnInterfaceName)));
1733                     } else {
1734                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1735                                 RouterInterfaces.class,
1736                                 new RouterInterfacesKey(vpnInterfaceName)));
1737                     }
1738                 }
1739             }
1740         }
1741     }
1742
1743     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1744                                                   WriteTransaction writeOperTxn) {
1745         if(dpId.equals(BigInteger.ZERO)) {
1746             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1747             return;
1748         }
1749         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1750         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1751                 .OPERATIONAL, routerDpnListIdentifier);
1752         if (optionalRouterDpnList.isPresent()) {
1753             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1754             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1755             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1756                 if (routerInterfaces.isEmpty()) {
1757                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1758                 } else {
1759                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1760                             RouterInterfaces.class,
1761                             new RouterInterfacesKey(vpnInterfaceName)));
1762                 }
1763             }
1764         }
1765     }
1766
1767     void notifyTaskIfRequired(String intfName) {
1768         Runnable notifyTask = vpnIntfMap.remove(intfName);
1769         if (notifyTask == null) {
1770             return;
1771         }
1772         executorService.execute(notifyTask);
1773     }
1774 }