vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Function;
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Preconditions;
  13 import com.google.common.base.Predicate;
  14 import com.google.common.base.Predicates;
  15 import com.google.common.collect.FluentIterable;
  16 import com.google.common.collect.Iterators;
  17 import com.google.common.util.concurrent.CheckedFuture;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.JdkFutureAdapters;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import java.math.BigInteger;
  23 import java.util.ArrayList;
  24 import java.util.Arrays;
  25 import java.util.Collection;
  26 import java.util.Iterator;
  27 import java.util.List;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ExecutionException;
  31 import java.util.concurrent.ExecutorService;
  32 import java.util.concurrent.Executors;
  33 import java.util.concurrent.Future;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  36 import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
  37 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  38 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  41 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  42 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  43 import org.opendaylight.genius.mdsalutil.ActionInfo;
  44 import org.opendaylight.genius.mdsalutil.ActionType;
  45 import org.opendaylight.genius.mdsalutil.FlowEntity;
  46 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  47 import org.opendaylight.genius.mdsalutil.InstructionType;
  48 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  49 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  50 import org.opendaylight.genius.mdsalutil.MatchInfo;
  51 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  52 import org.opendaylight.genius.mdsalutil.NwConstants;
  53 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  54 import org.opendaylight.genius.utils.ServiceIndex;
  55 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  56 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  57 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  58 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  59 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.VpnTargets;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInputBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEvent;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AddDpnEventBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEvent;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.RemoveDpnEventBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventData;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventDataBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventData;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventDataBuilder;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 131 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 132 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 133 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 134 import org.opendaylight.yangtools.yang.common.RpcError;
 135 import org.opendaylight.yangtools.yang.common.RpcResult;
 136 import org.slf4j.Logger;
 137 import org.slf4j.LoggerFactory;
 138
 139 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
 140     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 141     private ListenerRegistration<DataChangeListener> listenerRegistration;
 142     private final DataBroker dataBroker;
 143     private final IBgpManager bgpManager;
 144     private final IFibManager fibManager;
 145     private final IMdsalApiManager mdsalManager;
 146     private final IdManagerService idManager;
 147     private final OdlArputilService arpManager;
 148     private final OdlInterfaceRpcService ifaceMgrRpcService;
 149     private final NotificationPublishService notificationPublishService;
 150     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
 151     private ExecutorService executorService = Executors.newSingleThreadExecutor();
 152
 153     public VpnInterfaceManager(final DataBroker dataBroker,
 154                                final IBgpManager bgpManager,
 155                                final OdlArputilService arpManager,
 156                                final IdManagerService idManager,
 157                                final IMdsalApiManager mdsalManager,
 158                                final IFibManager fibManager,
 159                                final OdlInterfaceRpcService ifaceMgrRpcService,
 160                                final NotificationPublishService notificationPublishService) {
 161         super(VpnInterface.class);
 162         this.dataBroker = dataBroker;
 163         this.bgpManager = bgpManager;
 164         this.arpManager = arpManager;
 165         this.idManager = idManager;
 166         this.mdsalManager = mdsalManager;
 167         this.fibManager = fibManager;
 168         this.ifaceMgrRpcService = ifaceMgrRpcService;
 169         this.notificationPublishService = notificationPublishService;
 170     }
 171
 172     public void start() {
 173         LOG.info("{} start", getClass().getSimpleName());
 174         listenerRegistration = dataBroker.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 175                 getWildCardPath(), this, DataChangeScope.SUBTREE);
 176     }
 177
 178     private InstanceIdentifier<VpnInterface> getWildCardPath() {
 179         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 180     }
 181
 182     @Override
 183     public void close() throws Exception {
 184         if (listenerRegistration != null) {
 185             listenerRegistration.close();
 186             listenerRegistration = null;
 187         }
 188         LOG.info("{} close", getClass().getSimpleName());
 189     }
 190
 191     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 192         return InstanceIdentifier.create(InterfacesState.class)
 193                 .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 194     }
 195
 196     @Override
 197     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 198         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 199         LOG.info("VPN Interface add event - intfName {}" ,vpnInterface.getName());
 200         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 201         final String interfaceName = key.getName();
 202
 203         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 204                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 205         if(interfaceState != null){
 206             try{
 207                 final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 208                 final int ifIndex = interfaceState.getIfIndex();
 209                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 210                 dataStoreCoordinator.enqueueJob("VPNINTERFACE-"+ interfaceName,
 211                         new Callable<List<ListenableFuture<Void>>>() {
 212                             @Override
 213                             public List<ListenableFuture<Void>> call() throws Exception {
 214                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 215                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 216                                 WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 217                                 processVpnInterfaceUp(dpnId, vpnInterface, ifIndex, false, writeConfigTxn, writeOperTxn, writeInvTxn);
 218                                 List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
 219                                 futures.add(writeOperTxn.submit());
 220                                 futures.add(writeConfigTxn.submit());
 221                                 futures.add(writeInvTxn.submit());
 222                                 return futures;
 223                             }
 224                         });
 225             }catch (Exception e){
 226                 LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", interfaceName, e);
 227                 return;
 228             }
 229         } else {
 230             LOG.info("Handling addition of VPN interface {} skipped as interfaceState is not available", interfaceName);
 231         }
 232     }
 233
 234     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface,
 235                                          final int lPortTag, boolean isInterfaceUp,
 236                                          WriteTransaction writeConfigTxn,
 237                                          WriteTransaction writeOperTxn,
 238                                          WriteTransaction writeInvTxn) {
 239
 240         final String interfaceName = vpnInterface.getName();
 241         if (!isInterfaceUp) {
 242             final String vpnName = vpnInterface.getVpnInstanceName();
 243             LOG.info("Binding vpn service to interface {} ", interfaceName);
 244             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 245             if (vpnId == VpnConstants.INVALID_ID) {
 246                 LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 247                 return;
 248             }
 249             boolean waitForVpnInterfaceOpRemoval = false;
 250             VpnInterface opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, vpnInterface.getName());
 251             if (opVpnInterface != null ) {
 252                 String opVpnName = opVpnInterface.getVpnInstanceName();
 253                 String primaryInterfaceIp = null;
 254                 if(opVpnName.equals(vpnName)) {
 255                     // Please check if the primary VRF Entry does not exist for VPNInterface
 256                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 257                     // back to back
 258                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 259                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 260                     if (adjs == null) {
 261                         LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained", interfaceName);
 262                         return;
 263                     }
 264                     for (Adjacency adj : adjs) {
 265                         if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
 266                             primaryInterfaceIp = adj.getIpAddress();
 267                             break;
 268                         }
 269                     }
 270                     if (primaryInterfaceIp == null) {
 271                         LOG.info("VPN Interface {} addition failed as primary adjacency "
 272                                 + "for this vpn interface could not be obtained", interfaceName);
 273                         return;
 274                     }
 275                     // Get the rd of the vpn instance
 276                     String rd = getRouteDistinguisher(opVpnName);
 277                     rd =  (rd == null) ? opVpnName : rd;
 278                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, rd, primaryInterfaceIp);
 279                     if (vrf != null) {
 280                         LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
 281                         return;
 282                     }
 283                     waitForVpnInterfaceOpRemoval = true;
 284                 } else {
 285                     LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
 286                             interfaceName, vpnName, opVpnName);
 287                 }
 288             }
 289             if (!waitForVpnInterfaceOpRemoval) {
 290                 // Add the VPNInterface and quit
 291                 updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 292                 bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 293                 processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 294                 return;
 295             }
 296
 297             // FIB didn't get a chance yet to clean up this VPNInterface
 298             // Let us give it a chance here !
 299             LOG.info("Trying to add VPN Interface {}, but waiting for FIB to clean up! ", interfaceName);
 300             try {
 301                 Runnable notifyTask = new VpnNotifyTask();
 302                 vpnIntfMap.put(interfaceName, notifyTask);
 303                 synchronized (notifyTask) {
 304                     try {
 305                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 306                     } catch (InterruptedException e) {
 307                     }
 308                 }
 309             } finally {
 310                 vpnIntfMap.remove(interfaceName);
 311             }
 312
 313             opVpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
 314             if (opVpnInterface != null) {
 315                 LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
 316                 return;
 317             }
 318             // VPNInterface got removed, proceed with Add
 319             updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 320             bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 321             processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 322         } else {
 323             // Interface is retained in the DPN, but its Link Up.
 324             // Advertise prefixes again for this interface to BGP
 325             advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
 326                     vpnInterface);
 327         }
 328     }
 329
 330
 331 //    private class UpdateDpnToVpnWorker implements Callable<List<ListenableFuture<Void>>> {
 332 //        BigInteger dpnId;
 333 //        String vpnName;
 334 //        String interfaceName;
 335 //        boolean addToDpn;
 336 //        int lPortTag;
 337 //
 338 //        public UpdateDpnToVpnWorker(BigInteger dpnId, String vpnName, String interfaceName,
 339 //                                    int lPortTag, boolean addToDpn) {
 340 //            this.dpnId= dpnId;
 341 //            this.vpnName = vpnName;
 342 //            this.interfaceName = interfaceName;
 343 //            this.lPortTag = lPortTag;
 344 //            this.addToDpn = addToDpn;
 345 //        }
 346 //
 347 //        @Override
 348 //        public List<ListenableFuture<Void>> call() throws Exception {
 349 //            // If another renderer(for eg : CSS) needs to be supported, check can be performed here
 350 //            // to call the respective helpers.
 351 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 352 //            updateDpnDbs(dpnId, vpnName, interfaceName, addToDpn, writeTxn);
 353 //            List<ListenableFuture<Void>> futures = new ArrayList<>();
 354 //            futures.add(writeTxn.submit());
 355 //            ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
 356 //            Futures.addCallback(listenableFuture,
 357 //                    new UpdateDpnToVpnCallback(dpnId, vpnName, interfaceName, lPortTag, addToDpn));
 358 //            return futures;
 359 //        }
 360 //    }
 361 //
 362 //
 363 //    /**
 364 //     * JobCallback class is used as a future callback for
 365 //     * main and rollback workers to handle success and failure.
 366 //     */
 367 //    private class UpdateDpnToVpnCallback implements FutureCallback<List<Void>> {
 368 //        BigInteger dpnId;
 369 //        String vpnName;
 370 //        String interfaceName;
 371 //        boolean addToDpn;
 372 //        int lPortTag;
 373 //
 374 //        public UpdateDpnToVpnCallback(BigInteger dpnId, String vpnName, String interfaceName,
 375 //                                      int lPortTag, boolean addToDpn) {
 376 //            this.dpnId= dpnId;
 377 //            this.vpnName = vpnName;
 378 //            this.interfaceName = interfaceName;
 379 //            this.lPortTag = lPortTag;
 380 //            this.addToDpn = addToDpn;
 381 //        }
 382 //
 383 //        /**
 384 //         * @param voids
 385 //         * This implies that all the future instances have returned success. -- TODO: Confirm this
 386 //         */
 387 //        @Override
 388 //        public void onSuccess(List<Void> voids) {
 389 //            WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 390 //            bindService(dpnId, vpnName, interfaceName, lPortTag, writeTxn);
 391 //            processVpnInterfaceAdjacencies(dpnId, vpnName, interfaceName, writeTxn);
 392 //            writeTxn.submit();
 393 //        }
 394 //
 395 //        /**
 396 //         *
 397 //         * @param throwable
 398 //         * This method is used to handle failure callbacks.
 399 //         * If more retry needed, the retrycount is decremented and mainworker is executed again.
 400 //         * After retries completed, rollbackworker is executed.
 401 //         * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 402 //         */
 403 //
 404 //        @Override
 405 //        public void onFailure(Throwable throwable) {
 406 //            LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 407 //        }
 408 //    }
 409
 410
 411
 412
 413     private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
 414                                                  VpnInterface intf) {
 415         //Read NextHops
 416         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 417         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 418
 419         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 420         if (rd == null) {
 421             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 422                     intf.getName(), intf.getVpnInstanceName());
 423             return;
 424         } else {
 425             if (rd.equals(intf.getVpnInstanceName())) {
 426                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in " +
 427                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 428
 429                 return;
 430             }
 431         }
 432         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
 433                 intf.getVpnInstanceName(), rd);
 434
 435         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 436         if (nextHopIp == null){
 437             LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
 438             return;
 439         }
 440
 441         if (adjacencies.isPresent()) {
 442             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 443
 444             if (!nextHops.isEmpty()) {
 445                 LOG.trace("NextHops are " + nextHops);
 446                 for (Adjacency nextHop : nextHops) {
 447                     long label = nextHop.getLabel();
 448                     try {
 449                         LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 450                         bgpManager.advertisePrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
 451                         LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 452                     } catch(Exception e) {
 453                         LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
 454                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 455                     }
 456                 }
 457             }
 458         }
 459     }
 460
 461     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 462         //Read NextHops
 463         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 464         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 465
 466         String rd = VpnUtil.getVpnRd(dataBroker, intf.getVpnInstanceName());
 467         if (rd == null) {
 468             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 469                     intf.getName(), intf.getVpnInstanceName());
 470             return;
 471         } else {
 472             if (rd.equals(intf.getVpnInstanceName())) {
 473                 LOG.info("withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in " +
 474                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 475                 return;
 476             }
 477         }
 478         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 479                 intf.getVpnInstanceName(), rd);
 480         if (adjacencies.isPresent()) {
 481             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 482
 483             if (!nextHops.isEmpty()) {
 484                 LOG.trace("NextHops are " + nextHops);
 485                 for (Adjacency nextHop : nextHops) {
 486                     try {
 487                         LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 488                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 489                         LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 490                     } catch(Exception e) {
 491                         LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
 492                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 493                     }
 494                 }
 495             }
 496         }
 497     }
 498
 499     public void updateVpnToDpnMapping(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 500         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 501         if (dpId == null) {
 502             dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, interfaceName);
 503         }
 504         if(!dpId.equals(BigInteger.ZERO)) {
 505             if(add)
 506                 createOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 507             else
 508                 removeOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 509         }
 510     }
 511
 512     private void bindService(BigInteger dpId, final String vpnInstanceName, final String vpnInterfaceName,
 513                              int lPortTag, WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
 514         final int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 515         final long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
 516
 517         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 518         dataStoreCoordinator.enqueueJob(vpnInterfaceName,
 519                 new Callable<List<ListenableFuture<Void>>>() {
 520                     @Override
 521                     public List<ListenableFuture<Void>> call() throws Exception {
 522                         WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 523                         int instructionKey = 0;
 524                         List<Instruction> instructions = new ArrayList<Instruction>();
 525
 526                         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(
 527                                 BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 528                         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 529
 530                         BoundServices
 531                                 serviceInfo =
 532                                 InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 533                                         NwConstants.L3VPN_SERVICE_INDEX, priority,
 534                                         NwConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 535                         writeTxn.put(LogicalDatastoreType.CONFIGURATION,
 536                                 InterfaceUtils.buildServiceId(vpnInterfaceName, NwConstants.L3VPN_SERVICE_INDEX), serviceInfo, true);
 537                         List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
 538                         futures.add(writeTxn.submit());
 539                         return futures;
 540                     }
 541                 });
 542         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
 543                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW, writeInvTxn);
 544         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
 545                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.ADD_FLOW, writeInvTxn);
 546
 547     }
 548
 549     private void processVpnInterfaceAdjacencies(BigInteger dpnId, String vpnName, String interfaceName,
 550                                                 WriteTransaction writeConfigTxn,
 551                                                 WriteTransaction writeOperTxn) {
 552         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 553         // Read NextHops
 554         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 555         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 556
 557         if (adjacencies.isPresent()) {
 558             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 559             List<Adjacency> value = new ArrayList<>();
 560
 561             // Get the rd of the vpn instance
 562             String rd = getRouteDistinguisher(vpnName);
 563
 564             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 565             if (nextHopIp == null){
 566                 LOG.error("NextHop for interface {} is null", interfaceName);
 567                 return;
 568             }
 569
 570             List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
 571
 572             LOG.trace("NextHops for interface {} are {}", interfaceName, nextHops);
 573             for (Adjacency nextHop : nextHops) {
 574                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 575                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 576                         VpnUtil.getNextHopLabelKey((rd == null) ? vpnName
 577                                 : rd, prefix));
 578                 List<String> adjNextHop = nextHop.getNextHopIpList();
 579                 value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIpList(
 580                         (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : Arrays.asList(nextHopIp))
 581                         .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 582
 583                 if (nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 584                     LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, interfaceName, vpnName);
 585                     writeOperTxn.merge(
 586                             LogicalDatastoreType.OPERATIONAL,
 587                             VpnUtil.getPrefixToInterfaceIdentifier(
 588                                     VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 589                             VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix), true);
 590                 } else {
 591                     //Extra route adjacency
 592                     LOG.trace("Adding prefix {} and nexthopList {} as extra-route for vpn", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName);
 593                     writeOperTxn.merge(
 594                             LogicalDatastoreType.OPERATIONAL,
 595                             VpnUtil.getVpnToExtrarouteIdentifier(
 596                                     (rd != null) ? rd : vpnName, nextHop.getIpAddress()),
 597                             VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()), true);
 598                 }
 599             }
 600
 601             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 602
 603             VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 604             InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 605             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, true);
 606             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 607
 608             for (Adjacency nextHop : aug.getAdjacency()) {
 609                 long label = nextHop.getLabel();
 610                 if (rd != null) {
 611                     addToLabelMapper(label, dpnId, nextHop.getIpAddress(), Arrays.asList(nextHopIp), vpnId,
 612                             interfaceName, null,false, rd, writeOperTxn);
 613                     addPrefixToBGP(rd, nextHop.getIpAddress(), nextHopIp, label, writeConfigTxn);
 614                     //TODO: ERT - check for VPNs importing my route
 615                     for (VpnInstance vpn : vpnsToImportRoute) {
 616                         String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
 617                         if (vpnRd != null) {
 618                             LOG.debug("Exporting route with rd {} prefix {} nexthop {} label {} to VPN {}", vpnRd, nextHop.getIpAddress(), nextHopIp, label, vpn);
 619                             fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), Arrays.asList(nextHopIp), (int) label,
 620                                     RouteOrigin.SELF_IMPORTED, writeConfigTxn);
 621                         }
 622                     }
 623                 } else {
 624                     // ### add FIB route directly
 625                     fibManager.addOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), Arrays.asList(nextHopIp),
 626                             (int) label, RouteOrigin.STATIC, writeConfigTxn);
 627                 }
 628             }
 629         }
 630     }
 631
 632     private List<VpnInstance> getVpnsImportingMyRoute(final String vpnName) {
 633         List<VpnInstance> vpnsToImportRoute = new ArrayList<>();
 634
 635         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 636                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 637         Optional<VpnInstance> optVpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 638         final VpnInstance vpnInstance;
 639         if (optVpnInstance.isPresent()) {
 640             vpnInstance = optVpnInstance.get();
 641         } else {
 642             LOG.debug("Could not retrieve vpn instance {} to check for vpns importing the routes", vpnName);
 643             return vpnsToImportRoute;
 644         }
 645
 646         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 647             @Override
 648             public boolean apply(VpnInstance input) {
 649                 return !input.getVpnInstanceName().equals(vpnName);
 650             }
 651         };
 652
 653         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 654             @Override
 655             public boolean apply(VpnInstance input) {
 656                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ExportExtcommunity),
 657                         getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 658                 return Iterators.size(commonRTs.iterator()) > 0;
 659             }
 660         };
 661
 662         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 663             @Override
 664             public String apply(VpnInstance vpnInstance) {
 665                 //return vpnInstance.getVpnInstanceName();
 666                 return vpnInstance.getIpv4Family().getRouteDistinguisher();
 667             }
 668         };
 669
 670         vpnsToImportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(dataBroker)).
 671                 filter(excludeVpn).
 672                 filter(matchRTs).toList();
 673         return vpnsToImportRoute;
 674     }
 675
 676     private List<VpnInstance> getVpnsExportingMyRoute(final String vpnName) {
 677         List<VpnInstance> vpnsToExportRoute = new ArrayList<>();
 678
 679         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 680                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 681         Optional<VpnInstance> optVpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 682         final VpnInstance vpnInstance;
 683         if (optVpnInstance.isPresent()) {
 684             vpnInstance = optVpnInstance.get();
 685         } else {
 686             LOG.debug("Could not retrieve vpn instance {} to check for vpns exporting the routes", vpnName);
 687             return vpnsToExportRoute;
 688         }
 689
 690         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 691             @Override
 692             public boolean apply(VpnInstance input) {
 693                 return !input.getVpnInstanceName().equals(vpnName);
 694             }
 695         };
 696
 697         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 698             @Override
 699             public boolean apply(VpnInstance input) {
 700                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ImportExtcommunity),
 701                         getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
 702                 return Iterators.size(commonRTs.iterator()) > 0;
 703             }
 704         };
 705
 706         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 707             @Override
 708             public String apply(VpnInstance vpnInstance) {
 709                 return vpnInstance.getVpnInstanceName();
 710             }
 711         };
 712
 713         vpnsToExportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(dataBroker)).
 714                 filter(excludeVpn).
 715                 filter(matchRTs).toList();
 716         return vpnsToExportRoute;
 717     }
 718
 719     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
 720         final Predicate<T> inPredicate = Predicates.<T>in(collection2);
 721         return new Iterable<T>() {
 722             @Override
 723             public Iterator<T> iterator() {
 724                 return Iterators.filter(collection1.iterator(), inPredicate);
 725             }
 726         };
 727     }
 728
 729     private List<String> getRts(VpnInstance vpnInstance, VpnTarget.VrfRTType rtType) {
 730         String name = vpnInstance.getVpnInstanceName();
 731         List<String> rts = new ArrayList<>();
 732         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 733         if (vpnConfig == null) {
 734             LOG.trace("vpn config is not available for {}", name);
 735             return rts;
 736         }
 737         VpnTargets targets = vpnConfig.getVpnTargets();
 738         if (targets == null) {
 739             LOG.trace("vpn targets not available for {}", name);
 740             return rts;
 741         }
 742         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 743         if (vpnTargets == null) {
 744             LOG.trace("vpnTarget values not available for {}", name);
 745             return rts;
 746         }
 747         for (VpnTarget target : vpnTargets) {
 748             //TODO: Check for RT type is Both
 749             if(target.getVrfRTType().equals(rtType) ||
 750                     target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
 751                 String rtValue = target.getVrfRTValue();
 752                 rts.add(rtValue);
 753             }
 754         }
 755         return rts;
 756     }
 757
 758     private List<String> getExportRts(VpnInstance vpnInstance) {
 759         List<String> exportRts = new ArrayList<>();
 760         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 761         VpnTargets targets = vpnConfig.getVpnTargets();
 762         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 763         for (VpnTarget target : vpnTargets) {
 764             if (target.getVrfRTType().equals(VpnTarget.VrfRTType.ExportExtcommunity)) {
 765                 String rtValue = target.getVrfRTValue();
 766                 exportRts.add(rtValue);
 767             }
 768         }
 769         return exportRts;
 770     }
 771
 772     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 773                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow,
 774                              WriteTransaction writeConfigTxn){
 775         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 776         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, MetaDataUtil.getVpnIdMetadata(vpnId));
 777         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 778                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 779
 780         // Matching Arp reply flows
 781         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 782         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 783                 metadata, metadataMask }));
 784
 785         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 786
 787         // Instruction to punt to controller
 788         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 789         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 790         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 791         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
 792                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
 793
 794         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 795
 796         // Install the flow entry in L3_INTERFACE_TABLE
 797         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 798                 NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 799         FlowEntity flowEntity;
 800         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 801                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 802                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 803
 804         Flow flow = flowEntity.getFlowBuilder().build();
 805         String flowId = flowEntity.getFlowId();
 806         FlowKey flowKey = new FlowKey( new FlowId(flowId));
 807         Node nodeDpn = buildDpnNode(dpId);
 808
 809         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 810                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 811                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 812
 813         if (writeConfigTxn != null) {
 814             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 815                 LOG.debug("Creating ARP Flow for interface {}", vpnInterfaceName);
 816                 writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
 817             } else {
 818                 LOG.debug("Deleting ARP Flow for interface {}", vpnInterfaceName);
 819                 writeConfigTxn.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 820             }
 821         } else {
 822             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 823                 LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 824                 mdsalManager.installFlow(flowEntity);
 825             } else {
 826                 LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 827                 mdsalManager.removeFlow(flowEntity);
 828             }
 829         }
 830     }
 831
 832     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
 833     private Node buildDpnNode(BigInteger dpnId) {
 834         NodeId nodeId = new NodeId("openflow:" + dpnId);
 835         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
 836
 837         return nodeDpn;
 838     }
 839
 840     private String getRouteDistinguisher(String vpnName) {
 841         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 842                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 843         Optional<VpnInstance> vpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 844         String rd = "";
 845         if(vpnInstance.isPresent()) {
 846             VpnInstance instance = vpnInstance.get();
 847             VpnAfConfig config = instance.getIpv4Family();
 848             rd = config.getRouteDistinguisher();
 849         }
 850         return rd;
 851     }
 852
 853     /**
 854      * JobCallback class is used as a future callback for
 855      * main and rollback workers to handle success and failure.
 856      */
 857     private class DpnEnterExitVpnWorker implements FutureCallback<List<Void>> {
 858         BigInteger dpnId;
 859         String vpnName;
 860         String rd;
 861         boolean entered;
 862
 863         public DpnEnterExitVpnWorker(BigInteger dpnId, String vpnName, String rd, boolean entered) {
 864             this.entered = entered;
 865             this.dpnId = dpnId;
 866             this.vpnName = vpnName;
 867             this.rd = rd;
 868         }
 869
 870         /**
 871          * @param voids
 872          * This implies that all the future instances have returned success. -- TODO: Confirm this
 873          */
 874         @Override
 875         public void onSuccess(List<Void> voids) {
 876             if (entered) {
 877                 publishAddNotification(dpnId, vpnName, rd);
 878             } else {
 879                 publishRemoveNotification(dpnId, vpnName, rd);
 880             }
 881         }
 882
 883         /**
 884          *
 885          * @param throwable
 886          * This method is used to handle failure callbacks.
 887          * If more retry needed, the retrycount is decremented and mainworker is executed again.
 888          * After retries completed, rollbackworker is executed.
 889          * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 890          */
 891
 892         @Override
 893         public void onFailure(Throwable throwable) {
 894             LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 895         }
 896     }
 897
 898     private void createOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 899         String routeDistinguisher = getRouteDistinguisher(vpnName);
 900         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 901         Boolean newDpnOnVpn = Boolean.FALSE;
 902
 903         synchronized (vpnName.intern()) {
 904             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 905             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 906             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 907             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data
 908                     .entry.vpn.to.dpn.list.VpnInterfaces
 909                     vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 910
 911             if (dpnInVpn.isPresent()) {
 912                 VpnToDpnList vpnToDpnList = dpnInVpn.get();
 913                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 914                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = vpnToDpnList.getVpnInterfaces();
 915                 if (vpnInterfaces == null) {
 916                     vpnInterfaces = new ArrayList<>();
 917                 }
 918                 vpnInterfaces.add(vpnInterface);
 919                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder().setDpnId(dpnId);
 920                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 921                 if (writeTxn != null) {
 922                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 923                 } else {
 924                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 925                 }
 926                 /* If earlier state was inactive, it is considered new DPN coming back to the
 927                  * same VPN
 928                  */
 929                 if (vpnToDpnList.getDpnState() == VpnToDpnList.DpnState.Inactive) {
 930                     newDpnOnVpn = Boolean.TRUE;
 931                 }
 932             } else {
 933                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 934                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = new ArrayList<>();
 935                 vpnInterfaces.add(vpnInterface);
 936                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder().setDpnId(dpnId);
 937                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 938
 939                 if (writeTxn != null) {
 940                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 941                 } else {
 942                     VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 943                 }
 944                 newDpnOnVpn = Boolean.TRUE;
 945             }
 946             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
 947             try {
 948                 futures.get();
 949             } catch (InterruptedException | ExecutionException e) {
 950                 LOG.error("Error adding to dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
 951                 throw new RuntimeException(e.getMessage());
 952             }
 953
 954         }
 955         /*
 956          * Informing the Fib only after writeTxn is submitted successfuly.
 957          */
 958         if (newDpnOnVpn) {
 959             LOG.debug("Sending populateFib event for new dpn {} in VPN {}", dpnId, vpnName);
 960             fibManager.populateFibOnNewDpn(dpnId, vpnId, rd, new DpnEnterExitVpnWorker(dpnId, vpnName, rd, true /* entered */));
 961             publishAddNotification(dpnId, vpnName, rd);
 962         }
 963     }
 964
 965     private void removeOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 966         Boolean lastDpnOnVpn = Boolean.FALSE;
 967         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
 968         synchronized (vpnName.intern()) {
 969             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 970             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 971             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
 972             if (dpnInVpn.isPresent()) {
 973                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 974                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 975                 org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 976                         currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 977
 978                 if (vpnInterfaces.remove(currVpnInterface)) {
 979                     if (vpnInterfaces.isEmpty()) {
 980                         List<IpAddresses> ipAddresses = dpnInVpn.get().getIpAddresses();
 981                         if (ipAddresses == null || ipAddresses.isEmpty()) {
 982                             VpnToDpnListBuilder dpnInVpnBuilder =
 983                                     new VpnToDpnListBuilder(dpnInVpn.get())
 984                                             .setDpnState(VpnToDpnList.DpnState.Inactive)
 985                                             .setVpnInterfaces(null);
 986                             if (writeTxn != null) {
 987                                 writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build(), true);
 988                             } else {
 989                                 VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build());
 990                             }
 991                             lastDpnOnVpn = Boolean.TRUE;
 992                         } else {
 993                             LOG.warn("vpn interfaces are empty but ip addresses are present for the vpn {} in dpn {}", vpnName, dpnId);
 994                         }
 995                     } else {
 996                         if (writeTxn != null) {
 997                             writeTxn.delete(LogicalDatastoreType.OPERATIONAL, id.child(
 998                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 999                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1000                                     new VpnInterfacesKey(intfName)));
1001                         } else {
1002                             VpnUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1003                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1004                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1005                                     new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
1006                         }
1007                     }
1008                 }
1009             }
1010             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
1011             try {
1012                 futures.get();
1013             } catch (InterruptedException | ExecutionException e) {
1014                 LOG.error("Error removing from dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
1015                 throw new RuntimeException(e.getMessage());
1016             }
1017         }
1018         if (lastDpnOnVpn) {
1019             LOG.debug("Sending cleanup event for dpn {} in VPN {}", dpnId, vpnName);
1020             fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd, new DpnEnterExitVpnWorker(dpnId, vpnName, rd, false /* exited */));
1021             publishRemoveNotification(dpnId, vpnName, rd);
1022         }
1023     }
1024
1025     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1026         List<VpnInstance> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1027         for (VpnInstance vpn : vpnsToExportRoute) {
1028             String rd = vpn.getIpv4Family().getRouteDistinguisher();
1029             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getIpv4Family().getRouteDistinguisher());
1030             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1031             if (vrfEntries != null) {
1032                 for (VrfEntry vrfEntry : vrfEntries) {
1033                     try {
1034                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1035                             continue;
1036                         }
1037                         String prefix = vrfEntry.getDestPrefix();
1038                         long label = vrfEntry.getLabel();
1039                         List<String> nextHops = vrfEntry.getNextHopAddressList();
1040                         SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1041                         for (String nh : nextHops) {
1042                             if (route != null) {
1043                                 LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1044                                 importSubnetRouteForNewVpn(rd, prefix, nh, (int)label, route, writeConfigTxn);
1045                             } else {
1046                                 LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1047                                 fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, prefix, Arrays.asList(nh), (int)label,
1048                                         RouteOrigin.SELF_IMPORTED, writeConfigTxn);
1049                             }
1050                         }
1051                     } catch (Exception e) {
1052                         LOG.error("Exception occurred while importing route with prefix {} label {} nexthop {} from vpn {} to vpn {}", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpn.getVpnInstanceName(), vpnName);
1053                     }
1054                 }
1055                 writeConfigTxn.submit();
1056             } else {
1057                 LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getIpv4Family().getRouteDistinguisher());
1058             }
1059         }
1060     }
1061
1062     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label, WriteTransaction writeConfigTxn) {
1063         try {
1064             LOG.info("ADD: Adding Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1065             fibManager.addOrUpdateFibEntry(dataBroker, rd, prefix, Arrays.asList(nextHopIp), (int)label, RouteOrigin.STATIC, writeConfigTxn);
1066             bgpManager.advertisePrefix(rd, prefix, Arrays.asList(nextHopIp), (int)label);
1067             LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1068         } catch(Exception e) {
1069             LOG.error("Add prefix failed", e);
1070         }
1071     }
1072
1073     @Override
1074     public void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1075         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
1076         LOG.info("VPN Interface remove event - intfName {}" ,vpnInterface.getName());
1077         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1078         final String interfaceName = key.getName();
1079
1080         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1081         final Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1082         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
1083                 InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1084         if (optVpnInterface.isPresent()){
1085             BigInteger dpnId = BigInteger.ZERO;
1086             Boolean dpnIdRetrieved = Boolean.FALSE;
1087             if(interfaceState != null){
1088                 try{
1089                     dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1090                     dpnIdRetrieved = Boolean.TRUE;
1091                 }catch (Exception e){
1092                     LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", interfaceName, e);
1093                 }
1094             } else {
1095                 LOG.error("Unable to retrieve interfaceState for interface {} , quitting ", interfaceName);
1096                 return;
1097             }
1098             final VpnInterface vpnOpInterface = optVpnInterface.get();
1099             if(dpnIdRetrieved == Boolean.FALSE){
1100                 LOG.info("dpnId for {} has not been retrieved yet. Fetching from vpn interface operational DS", interfaceName);
1101                 dpnId = vpnOpInterface.getDpnId();
1102             }
1103             final int ifIndex = interfaceState.getIfIndex();
1104             final BigInteger dpId = dpnId;
1105             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1106             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1107                     new Callable<List<ListenableFuture<Void>>>() {
1108                         @Override
1109                         public List<ListenableFuture<Void>> call() throws Exception {
1110                             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1111                             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1112                             WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
1113                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, false, true, writeConfigTxn, writeOperTxn, writeInvTxn);
1114                             List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
1115                             futures.add(writeOperTxn.submit());
1116                             futures.add(writeConfigTxn.submit());
1117                             futures.add(writeInvTxn.submit());
1118                             return futures;
1119                         }
1120                     });
1121
1122         }else{
1123             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
1124         }
1125     }
1126
1127     protected void processVpnInterfaceDown(BigInteger dpId,
1128                                            String interfaceName,
1129                                            int lPortTag,
1130                                            boolean isInterfaceStateDown,
1131                                            boolean isConfigRemoval,
1132                                            WriteTransaction writeConfigTxn,
1133                                            WriteTransaction writeOperTxn,
1134                                            WriteTransaction writeInvTxn) {
1135         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1136         if (!isInterfaceStateDown) {
1137             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1138             if(vpnInterface == null){
1139                 LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
1140                 return;
1141             }else{
1142                 final String vpnName = vpnInterface.getVpnInstanceName();
1143                 if(!vpnInterface.isScheduledForRemove()){
1144                     VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE, writeOperTxn);
1145                     removeAdjacenciesFromVpn(dpId, interfaceName, vpnInterface.getVpnInstanceName(), writeConfigTxn);
1146                     LOG.info("Unbinding vpn service from interface {} ", interfaceName);
1147                     unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown, isConfigRemoval, writeConfigTxn, writeInvTxn);
1148                 }else{
1149                     LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ", interfaceName);
1150                     return;
1151                 }
1152             }
1153         } else {
1154             // Interface is retained in the DPN, but its Link Down.
1155             // Only withdraw the prefixes for this interface from BGP
1156             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(dataBroker, interfaceName);
1157             if(vpnInterface == null){
1158                 LOG.info("Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in operational data store", interfaceName);
1159                 return;
1160             }else {
1161                 withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
1162             }
1163         }
1164     }
1165
1166     private void waitForFibToRemoveVpnPrefix(String interfaceName) {
1167         // FIB didn't get a chance yet to clean up this VPNInterface
1168         // Let us give it a chance here !
1169         LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
1170         try {
1171             Runnable notifyTask = new VpnNotifyTask();
1172             vpnIntfMap.put(interfaceName, notifyTask);
1173             synchronized (notifyTask) {
1174                 try {
1175                     notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
1176                 } catch (InterruptedException e) {
1177                 }
1178             }
1179         } finally {
1180             vpnIntfMap.remove(interfaceName);
1181         }
1182     }
1183
1184     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final String interfaceName, final String vpnName,
1185                                           WriteTransaction writeConfigTxn) {
1186         //Read NextHops
1187         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1188         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1189         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1190
1191         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1192         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", interfaceName,
1193                 vpnName, rd);
1194         if (adjacencies.isPresent()) {
1195             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1196
1197             if (!nextHops.isEmpty()) {
1198                 LOG.trace("NextHops are " + nextHops);
1199                 for (Adjacency nextHop : nextHops) {
1200                     List<String> nhList = new ArrayList<String>();
1201                     if (nextHop.getMacAddress() == null || nextHop.getMacAddress().isEmpty()) {
1202                         // This is either an extra-route (or) a learned IP via subnet-route
1203                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1204                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1205                             LOG.error("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
1206                                     rd, nextHop.getIpAddress());
1207                             continue;
1208                         }
1209                         nhList = Arrays.asList(nextHopIp);
1210                     } else {
1211                         // This is a primary adjacency
1212                         nhList = nextHop.getNextHopIpList();
1213                     }
1214                     if (rd.equals(vpnName)) {
1215                         //this is an internal vpn - the rd is assigned to the vpn instance name;
1216                         //remove from FIB directly
1217                         for(String nh : nhList) {
1218                             fibManager.removeOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), nh, writeConfigTxn);
1219                         }
1220                     } else {
1221                         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1222                         for (String nh : nhList) {
1223                             //IRT: remove routes from other vpns importing it
1224                             removePrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1225                             for (VpnInstance vpn : vpnsToImportRoute) {
1226                                 String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
1227                                 if (vpnRd != null) {
1228                                     LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd, nextHop.getIpAddress(), vpn.getVpnInstanceName());
1229                                     fibManager.removeOrUpdateFibEntry(dataBroker, vpnRd, nextHop.getIpAddress(), nh, writeConfigTxn);
1230                                 }
1231                             }
1232                         }
1233                     }
1234                     String ip = nextHop.getIpAddress().split("/")[0];
1235                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
1236                             vpnName, ip);
1237                     if (vpnPortipToPort != null && !vpnPortipToPort.isConfig()) {
1238                         LOG.trace("VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
1239                                 vpnPortipToPort.getPortName(),ip);
1240                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip);
1241                     }
1242                 }
1243             }
1244         }
1245     }
1246
1247
1248     private void unbindService(BigInteger dpId, String vpnInstanceName, final String vpnInterfaceName,
1249                                int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval,
1250                                WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
1251         short l3vpn_service_index = ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX);
1252         if (!isInterfaceStateDown && isConfigRemoval) {
1253             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1254             dataStoreCoordinator.enqueueJob(vpnInterfaceName,
1255                     new Callable<List<ListenableFuture<Void>>>() {
1256                         @Override
1257                         public List<ListenableFuture<Void>> call() throws Exception {
1258                             WriteTransaction writeTxn = dataBroker.newWriteOnlyTransaction();
1259                             writeTxn.delete(LogicalDatastoreType.CONFIGURATION,
1260                                     InterfaceUtils.buildServiceId(vpnInterfaceName,
1261                                             NwConstants.L3VPN_SERVICE_INDEX));
1262
1263                             List<ListenableFuture<Void>> futures = new ArrayList<ListenableFuture<Void>>();
1264                             futures.add(writeTxn.submit());
1265                             return futures;
1266                         }
1267                     });
1268         }
1269         long vpnId = VpnUtil.getVpnId(dataBroker, vpnInstanceName);
1270         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
1271                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW, writeInvTxn);
1272         makeArpFlow(dpId, l3vpn_service_index, lPortTag, vpnInterfaceName,
1273                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.DEL_FLOW, writeInvTxn);
1274     }
1275
1276
1277     private void removePrefixFromBGP(String rd, String prefix, String nextHop, WriteTransaction writeConfigTxn) {
1278         try {
1279             LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
1280             fibManager.removeOrUpdateFibEntry(dataBroker, rd, prefix, nextHop, writeConfigTxn);
1281             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1282             LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
1283         } catch(Exception e) {
1284             LOG.error("Delete prefix failed", e);
1285         }
1286     }
1287
1288     @Override
1289     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
1290         LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
1291         LOG.info("VPN Interface update event - intfName {}" ,update.getName());
1292         String oldVpnName = original.getVpnInstanceName();
1293         String newVpnName = update.getVpnInstanceName();
1294         BigInteger dpnId = update.getDpnId();
1295         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
1296         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
1297         if (oldAdjs == null) {
1298             oldAdjs = new ArrayList<>();
1299         }
1300         if (newAdjs == null) {
1301             newAdjs = new ArrayList<>();
1302         }
1303         //handles switching between <internal VPN - external VPN>
1304         if (!oldVpnName.equals(newVpnName)) {
1305             remove(identifier, original);
1306             waitForFibToRemoveVpnPrefix(update.getName());
1307             add(identifier, update);
1308         }
1309         //handle both addition and removal of adjacencies
1310         //currently, new adjacency may be an extra route
1311         if (!oldAdjs.equals(newAdjs)) {
1312             for (Adjacency adj : newAdjs) {
1313                 if (oldAdjs.contains(adj)) {
1314                     oldAdjs.remove(adj);
1315                 } else {
1316                     // add new adjacency - right now only extra route will hit this path
1317                     addNewAdjToVpnInterface(identifier, adj, dpnId);
1318                 }
1319             }
1320             for (Adjacency adj : oldAdjs) {
1321                 delAdjFromVpnInterface(identifier, adj, dpnId);
1322             }
1323         }
1324     }
1325
1326     public void processArpRequest(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715
1327                                           .IpAddress srcIP, PhysAddress srcMac, org.opendaylight.yang.gen.v1.urn.ietf
1328             .params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress targetIP, PhysAddress targetMac, String srcInterface){
1329         //Build ARP response with ARP requests TargetIp TargetMac as the Arp Response SrcIp and SrcMac
1330         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
1331                 .setDstIpaddress(srcIP).setDstMacaddress(srcMac).setSrcIpaddress(targetIP).setSrcMacaddress(targetMac).build();
1332         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
1333         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
1334         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
1335             @Override
1336             public void onFailure(Throwable error) {
1337                 LOG.error("Error - {}", msgFormat, error);
1338             }
1339
1340             @Override
1341             public void onSuccess(RpcResult<Void> result) {
1342                 if(!result.isSuccessful()) {
1343                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
1344                 } else {
1345                     LOG.debug("Successful RPC Result - {}", msgFormat);
1346                 }
1347             }
1348         });
1349     }
1350
1351     private String getErrorText(Collection<RpcError> errors) {
1352         StringBuilder errorText = new StringBuilder();
1353         for(RpcError error : errors) {
1354             errorText.append(",").append(error.getErrorType()).append("-")
1355                     .append(error.getMessage());
1356         }
1357         return errorText.toString();
1358     }
1359
1360     private void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1361                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd,
1362                                   WriteTransaction writeOperTxn) {
1363         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1364         Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
1365         Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
1366         Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
1367         if (!isSubnetRoute) {
1368             // NextHop must be present for non-subnetroute entries
1369             Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
1370         }
1371         LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
1372         if (dpnId != null) {
1373             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1374                     .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
1375             LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1376             lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(vpnId)
1377                     .setIsSubnetRoute(isSubnetRoute);
1378             if (elanTag != null) {
1379                 lriBuilder.setElanTag(elanTag);
1380             }
1381             if (vpnInterfaceName != null) {
1382                 lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1383             }
1384             lriBuilder.setParentVpnRd(rd);
1385             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
1386             if (vpnInstanceOpDataEntry != null) {
1387                 List<String> vpnInstanceNames = Arrays.asList(vpnInstanceOpDataEntry.getVpnInstanceName());
1388                 lriBuilder.setVpnInstanceList(vpnInstanceNames);
1389             }
1390             LabelRouteInfo lri = lriBuilder.build();
1391             LOG.trace("Adding route info to label map: {}", lri);
1392             if (writeOperTxn != null) {
1393                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, lriIid, lri, true);
1394             } else {
1395                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
1396             }
1397         } else {
1398             LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
1399         }
1400     }
1401
1402     public synchronized void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
1403                                                         long elantag, BigInteger dpnId, WriteTransaction writeTxn) {
1404         SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
1405         RouteOrigin origin = RouteOrigin.STATIC; // Only case when a route is considered as directly connected
1406         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1407                 .setLabel((long)label).setOrigin(origin.getValue())
1408                 .addAugmentation(SubnetRoute.class, route).build();
1409
1410         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
1411
1412         //TODO: What should be parentVpnId? Get it from RD?
1413         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1414         addToLabelMapper((long)label, dpnId, prefix, Arrays.asList(nextHop), vpnId, null, elantag, true, rd, null);
1415         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1416
1417         InstanceIdentifierBuilder<VrfTables> idBuilder =
1418                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1419         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1420
1421         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1422                 setVrfEntry(vrfEntryList).build();
1423
1424         if (writeTxn != null) {
1425             writeTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1426         } else {
1427             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1428         }
1429
1430         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1431         if (vpnsToImportRoute.size() > 0) {
1432             VrfEntry importingVrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1433                     .setLabel((long) label).setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
1434                     .addAugmentation(SubnetRoute.class, route).build();
1435             List<VrfEntry> importingVrfEntryList = Arrays.asList(importingVrfEntry);
1436             for (VpnInstance vpnInstance : vpnsToImportRoute) {
1437                 LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop, label, vpnInstance.getVpnInstanceName());
1438                 String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1439                 InstanceIdentifier<VrfTables> importingVrfTableId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(importingRd)).build();
1440                 VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(importingVrfEntryList).build();
1441                 if (writeTxn != null) {
1442                     writeTxn.merge(LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable, true);
1443                 } else {
1444                     VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable);
1445                 }
1446             }
1447         }
1448     }
1449
1450     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1451                                                         SubnetRoute route, WriteTransaction writeConfigTxn) {
1452
1453         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1454         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1455                 .setLabel((long)label).setOrigin(origin.getValue())
1456                 .addAugmentation(SubnetRoute.class, route).build();
1457         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, route.getElantag());
1458         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1459         InstanceIdentifierBuilder<VrfTables> idBuilder =
1460                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1461         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1462         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1463                 setVrfEntry(vrfEntryList).build();
1464         if (writeConfigTxn != null) {
1465             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1466         } else {
1467             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1468         }
1469     }
1470
1471     public synchronized void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName){
1472         fibManager.removeOrUpdateFibEntry(dataBroker, rd, prefix, null /* nextHopToRemove */, null);
1473         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1474         for (VpnInstance vpnInstance : vpnsToImportRoute) {
1475             String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1476             LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix, vpnInstance.getVpnInstanceName());
1477             fibManager.removeOrUpdateFibEntry(dataBroker, importingRd, prefix, null /* nextHopToRemove */, null);
1478         }
1479     }
1480
1481     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1482
1483         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1484
1485         if (optVpnInterface.isPresent()) {
1486             VpnInterface currVpnIntf = optVpnInterface.get();
1487             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1488             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1489             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1490             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, adjPath);
1491             long label =
1492                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1493                             VpnUtil.getNextHopLabelKey((rd != null) ? rd : currVpnIntf.getVpnInstanceName(), prefix));
1494
1495             List<Adjacency> adjacencies;
1496             if (optAdjacencies.isPresent()) {
1497                 adjacencies = optAdjacencies.get().getAdjacency();
1498             } else {
1499                 //This code will not be hit since VM adjacency will always be there
1500                 adjacencies = new ArrayList<>();
1501             }
1502
1503             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIpList(adj.getNextHopIpList())
1504                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
1505
1506             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1507             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove());
1508
1509             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1510             for (String nh : adj.getNextHopIpList()) {
1511                 addExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(), (int) label,
1512                         currVpnIntf.getName());
1513             }
1514         }
1515     }
1516
1517     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1518         Optional<VpnInterface> optVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1519
1520         if (optVpnInterface.isPresent()) {
1521             VpnInterface currVpnIntf = optVpnInterface.get();
1522
1523             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1524             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1525             if (optAdjacencies.isPresent()) {
1526                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1527
1528                 if (!adjacencies.isEmpty()) {
1529                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1530                     LOG.trace("Adjacencies are " + adjacencies);
1531                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1532                     while (adjIt.hasNext()) {
1533                         Adjacency adjElem = adjIt.next();
1534                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1535                             adjIt.remove();
1536
1537                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1538                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1539                                     currVpnIntf.getVpnInstanceName(),
1540                                     aug, dpnId, currVpnIntf.isScheduledForRemove());
1541
1542                             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1543
1544                             for (String nh : adj.getNextHopIpList()) {
1545                                 delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
1546                                         currVpnIntf.getName());
1547                             }
1548                             break;
1549                         }
1550
1551                     }
1552                 }
1553             }
1554         }
1555
1556     }
1557
1558     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label,
1559                                  String intfName) {
1560
1561         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1562         VpnUtil.syncUpdate(
1563                 dataBroker,
1564                 LogicalDatastoreType.OPERATIONAL,
1565                 VpnUtil.getVpnToExtrarouteIdentifier( (rd != null) ? rd : routerID, destination),
1566                 VpnUtil.getVpnToExtraroute(destination, Arrays.asList(nextHop)));
1567
1568         BigInteger dpnId = null;
1569         if (intfName != null && !intfName.isEmpty()) {
1570             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1571             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1572             if (nextHopIp == null || nextHopIp.isEmpty()) {
1573                 LOG.error("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1574                         intfName, destination);
1575                 return;
1576             }
1577             nextHop = nextHopIp;
1578         }
1579         List<String> nextHopIpList = Arrays.asList(nextHop);
1580         if (rd != null) {
1581             /* Label mapper is required only for BGP VPN and not for Internal VPN */
1582             addToLabelMapper((long) label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(dataBroker, routerID),
1583                     intfName, null, false, rd, null);
1584         }
1585
1586         // TODO (eperefr): This is a limitation to be stated in docs. When configuring static route to go to
1587         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1588         // first place.
1589         InterVpnLink interVpnLink = VpnUtil.getInterVpnLinkByEndpointIp(dataBroker, nextHop);
1590         if ( interVpnLink != null ) {
1591             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1592             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1593             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1594             // This is like leaking one of the Vpn2 routes towards Vpn1
1595             boolean nexthopIsVpn2 = ( interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nextHop) );
1596             String srcVpnUuid = (nexthopIsVpn2) ? interVpnLink.getSecondEndpoint().getVpnUuid().getValue()
1597                     : interVpnLink.getFirstEndpoint().getVpnUuid().getValue();
1598             String dstVpnUuid = (nexthopIsVpn2) ? interVpnLink.getFirstEndpoint().getVpnUuid().getValue()
1599                     : interVpnLink.getSecondEndpoint().getVpnUuid().getValue();
1600             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
1601             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1602                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1603             VpnUtil.leakRoute(dataBroker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel);
1604         } else {
1605             if (rd != null) {
1606                 addPrefixToBGP(rd, destination, nextHop, label, null);
1607             } else {
1608                 // ### add FIB route directly
1609                 fibManager.addOrUpdateFibEntry(dataBroker, routerID, destination, Arrays.asList(nextHop), label, RouteOrigin.STATIC, null);
1610             }
1611         }
1612     }
1613
1614     protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName) {
1615         if (intfName != null && !intfName.isEmpty()) {
1616             BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1617             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1618             if (nextHopIp == null || nextHopIp.isEmpty()) {
1619                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1620                         intfName, destination);
1621             }
1622             nextHop = nextHopIp;
1623         }
1624
1625         if (rd != null) {
1626             removePrefixFromBGP(rd, destination, nextHop, null);
1627         } else {
1628             // ### add FIB route directly
1629             fibManager.removeOrUpdateFibEntry(dataBroker, routerID, destination, nextHop, null);
1630         }
1631     }
1632
1633     void publishAddNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1634         LOG.debug("Sending notification for add dpn {} in vpn {} event ", dpnId, vpnName);
1635         AddEventData data = new AddEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1636         AddDpnEvent event = new AddDpnEventBuilder().setAddEventData(data).build();
1637         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1638         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1639             @Override
1640             public void onFailure(Throwable error) {
1641                 LOG.warn("Error in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName, error);
1642             }
1643
1644             @Override
1645             public void onSuccess(Object arg) {
1646                 LOG.trace("Successful in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName);
1647             }
1648         });
1649     }
1650
1651     void publishRemoveNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1652         LOG.debug("Sending notification for remove dpn {} in vpn {} event ", dpnId, vpnName);
1653         RemoveEventData data = new RemoveEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1654         RemoveDpnEvent event = new RemoveDpnEventBuilder().setRemoveEventData(data).build();
1655         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1656         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1657             @Override
1658             public void onFailure(Throwable error) {
1659                 LOG.warn("Error in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName, error);
1660             }
1661
1662             @Override
1663             public void onSuccess(Object arg) {
1664                 LOG.trace("Successful in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName);
1665             }
1666         });
1667     }
1668
1669     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1670         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1671                 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1672                 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1673     }
1674
1675     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1676         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1677                 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1678     }
1679
1680     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1681         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1682         if(dpId.equals(BigInteger.ZERO)) {
1683             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
1684             return;
1685         }
1686         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1687
1688         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1689                 .OPERATIONAL, routerDpnListIdentifier);
1690         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1691         if (optionalRouterDpnList.isPresent()) {
1692             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1693                     RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1694         } else {
1695             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1696             builder.setRouterId(routerName);
1697             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1698             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
1699             routerInterfaces.add(routerInterface);
1700             builder.setDpnVpninterfacesList(Arrays.asList(dpnVpnList.build()));
1701             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1702                     getRouterId(routerName),
1703                     builder.build(), true);
1704         }
1705     }
1706
1707     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1708         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1709         if(dpId.equals(BigInteger.ZERO)) {
1710             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1711             return;
1712         }
1713         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1714         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1715                 .OPERATIONAL, routerDpnListIdentifier);
1716         if (optionalRouterDpnList.isPresent()) {
1717             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1718             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1719
1720             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1721                 if (routerInterfaces.isEmpty()) {
1722                     if (writeOperTxn != null) {
1723                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1724                     } else {
1725                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1726                     }
1727                 } else {
1728                     if (writeOperTxn != null) {
1729                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1730                                 RouterInterfaces.class,
1731                                 new RouterInterfacesKey(vpnInterfaceName)));
1732                     } else {
1733                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1734                                 RouterInterfaces.class,
1735                                 new RouterInterfacesKey(vpnInterfaceName)));
1736                     }
1737                 }
1738             }
1739         }
1740     }
1741
1742     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1743                                                   WriteTransaction writeOperTxn) {
1744         if(dpId.equals(BigInteger.ZERO)) {
1745             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1746             return;
1747         }
1748         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1749         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1750                 .OPERATIONAL, routerDpnListIdentifier);
1751         if (optionalRouterDpnList.isPresent()) {
1752             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1753             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1754             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1755                 if (routerInterfaces.isEmpty()) {
1756                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1757                 } else {
1758                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1759                             RouterInterfaces.class,
1760                             new RouterInterfacesKey(vpnInterfaceName)));
1761                 }
1762             }
1763         }
1764     }
1765
1766     public void addMIPAdjacency(String vpnName,String vpnInterface, org.opendaylight.yang.gen.v1.urn.ietf.params.xml
1767             .ns.yang.ietf.inet.types.rev130715.IpAddress prefix){
1768
1769         LOG.trace("Adding {} adjacency to VPN Interface {} ",prefix,vpnInterface);
1770         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
1771         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
1772         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
1773         String nextHopIpAddr = null;
1774         String nextHopMacAddress = null;
1775         String ip = prefix.getIpv4Address().getValue();
1776         if (adjacencies.isPresent()) {
1777             List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
1778             ip = VpnUtil.getIpPrefix(ip);
1779             for (Adjacency adjacs : adjacencyList) {
1780                 if (adjacs.getMacAddress() != null && !adjacs.getMacAddress().isEmpty()) {
1781                     nextHopIpAddr = adjacs.getIpAddress();
1782                     nextHopMacAddress = adjacs.getMacAddress();
1783                     break;
1784                 }
1785             }
1786             if (nextHopMacAddress != null && ip != null) {
1787                 String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1788                 long label =
1789                         VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1790                                 VpnUtil.getNextHopLabelKey((rd != null) ? rd : vpnName, ip));
1791                 String nextHopIp = nextHopIpAddr.split("/")[0];
1792                 Adjacency newAdj = new AdjacencyBuilder().setIpAddress(ip).setKey
1793                         (new AdjacencyKey(ip)).setNextHopIpList(Arrays.asList(nextHopIp)).build();
1794                 adjacencyList.add(newAdj);
1795                 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
1796                 VpnInterface newVpnIntf = new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(vpnInterface)).
1797                         setName(vpnInterface).setVpnInstanceName(vpnName).addAugmentation(Adjacencies.class, aug).build();
1798                 VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vpnIfId, newVpnIntf);
1799                 LOG.debug(" Successfully stored subnetroute Adjacency into VpnInterface {}", vpnInterface);
1800             }
1801         }
1802
1803     }
1804
1805     public void removeMIPAdjacency(String vpnName, String vpnInterface, org.opendaylight.yang.gen.v1.urn.ietf.params
1806             .xml.ns.yang.ietf.inet.types.rev130715.IpAddress prefix) {
1807         String ip = VpnUtil.getIpPrefix(prefix.getIpv4Address().getValue());
1808         LOG.trace("Removing {} adjacency from Old VPN Interface {} ",ip,vpnInterface);
1809         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
1810         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
1811         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1812         if (adjacencies.isPresent()) {
1813             InstanceIdentifier<Adjacency> adjacencyIdentifier = InstanceIdentifier.builder(VpnInterfaces.class).
1814                     child(VpnInterface.class, new VpnInterfaceKey(vpnInterface)).augmentation(Adjacencies.class)
1815                     .child(Adjacency.class, new AdjacencyKey(ip)).build();
1816             MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, adjacencyIdentifier);
1817             LOG.trace("Successfully Deleted Adjacency into VpnInterface {}", vpnInterface);
1818         }
1819     }
1820
1821     void notifyTaskIfRequired(String intfName) {
1822         Runnable notifyTask = vpnIntfMap.remove(intfName);
1823         if (notifyTask == null) {
1824             return;
1825         }
1826         executorService.execute(notifyTask);
1827     }
1828 }