vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.*;
  11 import com.google.common.collect.FluentIterable;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.*;
  14 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  15 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  16 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  17 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  18 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  19 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  20
  21 import org.opendaylight.controller.md.sal.binding.api.*;
  22 import org.opendaylight.genius.mdsalutil.*;
  23 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  24 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.VpnTargets;
  25 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  26 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeExternal;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeHwvtep;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeInternal;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.IsDcgwPresentInputBuilder;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.IsDcgwPresentOutput;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.*;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventData;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventDataBuilder;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventData;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventDataBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  71
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
  73         .VpnInstanceOpDataEntryBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInput;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInputBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  90
  91 import java.math.BigInteger;
  92 import java.util.Collection;
  93 import java.util.List;
  94 import java.util.ArrayList;
  95 import java.util.Arrays;
  96 import java.util.Iterator;
  97 import java.util.concurrent.*;
  98
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 102 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 106 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 108 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 109 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 110 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
 111 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
 115 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
 116 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
 117 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
 118 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
 119 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
 120 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
 121 import org.opendaylight.yangtools.yang.common.RpcError;
 122 import org.opendaylight.yangtools.yang.common.RpcResult;
 123 import org.slf4j.Logger;
 124 import org.slf4j.LoggerFactory;
 125
 126 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
 127     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 128     private ListenerRegistration<DataChangeListener> listenerRegistration, opListenerRegistration, tunnelInterfaceStateListenerRegistration;
 129     private final DataBroker broker;
 130     private final IBgpManager bgpManager;
 131     private IFibManager fibManager;
 132     private IMdsalApiManager mdsalManager;
 133     private OdlInterfaceRpcService ifaceMgrRpcService;
 134     private ItmRpcService itmProvider;
 135     private IdManagerService idManager;
 136     private OdlArputilService arpManager;
 137     private NeutronvpnService neuService;
 138     private VpnSubnetRouteHandler vpnSubnetRouteHandler;
 139     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
 140     private ExecutorService executorService = Executors.newSingleThreadExecutor();
 141     private InterfaceStateChangeListener interfaceListener;
 142     private SubnetRouteInterfaceStateChangeListener subnetRouteInterfaceListener;
 143     private TunnelInterfaceStateListener tunnelInterfaceStateListener;
 144     private VpnInterfaceOpListener vpnInterfaceOpListener;
 145     private ArpNotificationHandler arpNotificationHandler;
 146     private DpnInVpnChangeListener dpnInVpnChangeListener;
 147     private NotificationPublishService notificationPublishService;
 148     private FibRpcService fibService;
 149
 150     /**
 151      * Responsible for listening to data change related to VPN Interface
 152      * Bind VPN Service on the interface and informs the BGP service
 153      *
 154      * @param db - dataBroker service reference
 155      * @param bgpManager Used to advertise routes to the BGP Router
 156      * @param notificationService Used to subscribe to notification events
 157      */
 158     public VpnInterfaceManager(final DataBroker db, final IBgpManager bgpManager, NotificationService notificationService) {
 159         super(VpnInterface.class);
 160         broker = db;
 161         this.bgpManager = bgpManager;
 162         interfaceListener = new InterfaceStateChangeListener(db, this);
 163         subnetRouteInterfaceListener = new SubnetRouteInterfaceStateChangeListener(db, this);
 164         vpnInterfaceOpListener = new VpnInterfaceOpListener();
 165         arpNotificationHandler = new ArpNotificationHandler(this, broker);
 166         vpnSubnetRouteHandler = new VpnSubnetRouteHandler(broker, bgpManager, this);
 167         dpnInVpnChangeListener = new DpnInVpnChangeListener(broker);
 168         tunnelInterfaceStateListener = new TunnelInterfaceStateListener(broker, bgpManager, fibManager);
 169         notificationService.registerNotificationListener(vpnSubnetRouteHandler);
 170         notificationService.registerNotificationListener(arpNotificationHandler);
 171         notificationService.registerNotificationListener(dpnInVpnChangeListener);
 172         registerListener(db);
 173     }
 174
 175     public void setMdsalManager(IMdsalApiManager mdsalManager) {
 176         this.mdsalManager = mdsalManager;
 177     }
 178
 179     public void setIfaceMgrRpcService(OdlInterfaceRpcService ifMgrRpcService) {
 180         this.ifaceMgrRpcService = ifMgrRpcService;
 181         interfaceListener.setIfaceMgrRpcService(ifMgrRpcService);
 182     }
 183
 184     public void setFibManager(IFibManager fibManager) {
 185         this.fibManager = fibManager;
 186     }
 187
 188     public IFibManager getFibManager() {
 189         return this.fibManager;
 190     }
 191
 192
 193     public void setIdManager(IdManagerService idManager) {
 194         this.idManager = idManager;
 195         vpnSubnetRouteHandler.setIdManager(idManager);
 196     }
 197
 198     public void setArpManager(OdlArputilService arpManager) {
 199         this.arpManager = arpManager;
 200     }
 201
 202     void setNotificationPublishService(NotificationPublishService notificationPublishService) {
 203         this.notificationPublishService = notificationPublishService;
 204     }
 205
 206     public void setNeutronvpnManager(NeutronvpnService neuService) { this.neuService = neuService; }
 207
 208     public void setFibRpcService(FibRpcService fibService) {
 209         this.fibService = fibService;
 210     }
 211
 212     public FibRpcService getFibRpcService() {
 213         return fibService;
 214     }
 215
 216     public VpnSubnetRouteHandler getVpnSubnetRouteHandler() {
 217         return this.vpnSubnetRouteHandler;
 218     }
 219
 220     @Override
 221     public void close() throws Exception {
 222         if (listenerRegistration != null) {
 223             try {
 224                 listenerRegistration.close();
 225                 opListenerRegistration.close();
 226             } catch (final Exception e) {
 227                 LOG.error("Error when cleaning up DataChangeListener.", e);
 228             }
 229             listenerRegistration = null;
 230             opListenerRegistration = null;
 231         }
 232         LOG.info("VPN Interface Manager Closed");
 233     }
 234
 235     private void registerListener(final DataBroker db) {
 236         try {
 237             listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 238                     getWildCardPath(), VpnInterfaceManager.this, DataChangeScope.SUBTREE);
 239             opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
 240                     getWildCardPath(), vpnInterfaceOpListener, DataChangeScope.SUBTREE);
 241             tunnelInterfaceStateListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
 242                     getTunnelInterfaceStateListenerPath(), tunnelInterfaceStateListener, DataChangeScope.SUBTREE);
 243         } catch (final Exception e) {
 244             LOG.error("VPN Service DataChange listener registration fail!", e);
 245             throw new IllegalStateException("VPN Service registration Listener failed.", e);
 246         }
 247     }
 248
 249     private InstanceIdentifier<StateTunnelList> getTunnelInterfaceStateListenerPath() {
 250         return InstanceIdentifier.create(TunnelsState.class).child(StateTunnelList.class);
 251     }
 252
 253     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 254         return InstanceIdentifier.create(InterfacesState.class)
 255                 .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 256     }
 257
 258     @Override
 259     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 260         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 261         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 262         final String interfaceName = key.getName();
 263
 264         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 265                 InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 266         if(interfaceState != null){
 267             try{
 268                 final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 269                 final int ifIndex = interfaceState.getIfIndex();
 270                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 271                 dataStoreCoordinator.enqueueJob("VPNINTERFACE-"+ interfaceName,
 272                         new Callable<List<ListenableFuture<Void>>>() {
 273                             @Override
 274                             public List<ListenableFuture<Void>> call() throws Exception {
 275                                 WriteTransaction writeConfigTxn = broker.newWriteOnlyTransaction();
 276                                 WriteTransaction writeOperTxn = broker.newWriteOnlyTransaction();
 277                                 processVpnInterfaceUp(dpnId, vpnInterface, ifIndex, false, writeConfigTxn, writeOperTxn);
 278                                 CheckedFuture<Void, TransactionCommitFailedException> futures = writeOperTxn.submit();
 279                                 try {
 280                                     futures.get();
 281                                 } catch (InterruptedException | ExecutionException e) {
 282                                     LOG.error("Error adding oper data for interface {} to vpn {} on dpn {}", interfaceName,
 283                                             vpnInterface.getVpnInstanceName(), dpnId);
 284                                     throw new RuntimeException(e.getMessage());
 285                                 }
 286                                 futures = writeConfigTxn.submit();
 287                                 try {
 288                                     futures.get();
 289                                 } catch (InterruptedException | ExecutionException e) {
 290                                     LOG.error("Error adding config data for interface {} to vpn {} on dpn {}", interfaceName,
 291                                             vpnInterface.getVpnInstanceName(), dpnId);
 292                                     throw new RuntimeException(e.getMessage());
 293                                 }
 294                                 return null;
 295                             }
 296                         });
 297             }catch (Exception e){
 298                 LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", interfaceName, e);
 299                 return;
 300             }
 301         } else {
 302             LOG.info("Handling addition of VPN interface {} skipped as interfaceState is not available", interfaceName);
 303         }
 304     }
 305
 306     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface,
 307                                          final int lPortTag, boolean isInterfaceUp,
 308                                          WriteTransaction writeConfigTxn,
 309                                          WriteTransaction writeOperTxn) {
 310
 311         final String interfaceName = vpnInterface.getName();
 312         if (!isInterfaceUp) {
 313             final String vpnName = vpnInterface.getVpnInstanceName();
 314             LOG.info("Binding vpn service to interface {} ", interfaceName);
 315             long vpnId = VpnUtil.getVpnId(broker, vpnName);
 316             if (vpnId == VpnConstants.INVALID_ID) {
 317                 LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 318                 return;
 319             }
 320             boolean waitForVpnInterfaceOpRemoval = false;
 321             VpnInterface opVpnInterface = VpnUtil.getOperationalVpnInterface(broker, vpnInterface.getName());
 322             if (opVpnInterface != null ) {
 323                 String opVpnName = opVpnInterface.getVpnInstanceName();
 324                 String primaryInterfaceIp = null;
 325                 if(opVpnName.equals(vpnName)) {
 326                     // Please check if the primary VRF Entry does not exist for VPNInterface
 327                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 328                     // back to back
 329                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 330                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(broker, interfaceName);
 331                     if (adjs == null) {
 332                         LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained", interfaceName);
 333                         return;
 334                     }
 335                     for (Adjacency adj : adjs) {
 336                         if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
 337                             primaryInterfaceIp = adj.getIpAddress();
 338                             break;
 339                         }
 340                     }
 341                     if (primaryInterfaceIp == null) {
 342                         LOG.info("VPN Interface {} addition failed as primary adjacency "
 343                                 + "for this vpn interface could not be obtained", interfaceName);
 344                         return;
 345                     }
 346                     // Get the rd of the vpn instance
 347                     String rd = getRouteDistinguisher(opVpnName);
 348                     VrfEntry vrf = VpnUtil.getVrfEntry(broker, rd, primaryInterfaceIp);
 349                     if (vrf != null) {
 350                         LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
 351                         return;
 352                     }
 353                     waitForVpnInterfaceOpRemoval = true;
 354                 } else {
 355                     LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
 356                             interfaceName, vpnName, opVpnName);
 357                 }
 358             }
 359             if (!waitForVpnInterfaceOpRemoval) {
 360                 // Add the VPNInterface and quit
 361                 updateDpnDbs(dpId, vpnName, interfaceName, true /* add */);
 362                 bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn);
 363                 processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 364                 return;
 365             }
 366
 367             // FIB didn't get a chance yet to clean up this VPNInterface
 368             // Let us give it a chance here !
 369             LOG.info("VPN Interface {} waiting for FIB to clean up! ", interfaceName);
 370             try {
 371                 Runnable notifyTask = new VpnNotifyTask();
 372                 vpnIntfMap.put(interfaceName, notifyTask);
 373                 synchronized (notifyTask) {
 374                     try {
 375                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 376                     } catch (InterruptedException e) {
 377                     }
 378                 }
 379             } finally {
 380                 vpnIntfMap.remove(interfaceName);
 381             }
 382
 383             opVpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
 384             if (opVpnInterface != null) {
 385                 LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
 386                 return;
 387             }
 388             // VPNInterface got removed, proceed with Add
 389             updateDpnDbs(dpId, vpnName, interfaceName, true /* add */);
 390             bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn);
 391             processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 392         } else {
 393             // Interface is retained in the DPN, but its Link Up.
 394             // Advertise prefixes again for this interface to BGP
 395             advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
 396                     vpnInterface);
 397         }
 398     }
 399
 400
 401 //    private class UpdateDpnToVpnWorker implements Callable<List<ListenableFuture<Void>>> {
 402 //        BigInteger dpnId;
 403 //        String vpnName;
 404 //        String interfaceName;
 405 //        boolean addToDpn;
 406 //        int lPortTag;
 407 //
 408 //        public UpdateDpnToVpnWorker(BigInteger dpnId, String vpnName, String interfaceName,
 409 //                                    int lPortTag, boolean addToDpn) {
 410 //            this.dpnId= dpnId;
 411 //            this.vpnName = vpnName;
 412 //            this.interfaceName = interfaceName;
 413 //            this.lPortTag = lPortTag;
 414 //            this.addToDpn = addToDpn;
 415 //        }
 416 //
 417 //        @Override
 418 //        public List<ListenableFuture<Void>> call() throws Exception {
 419 //            // If another renderer(for eg : CSS) needs to be supported, check can be performed here
 420 //            // to call the respective helpers.
 421 //            WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
 422 //            updateDpnDbs(dpnId, vpnName, interfaceName, addToDpn, writeTxn);
 423 //            List<ListenableFuture<Void>> futures = new ArrayList<>();
 424 //            futures.add(writeTxn.submit());
 425 //            ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
 426 //            Futures.addCallback(listenableFuture,
 427 //                    new UpdateDpnToVpnCallback(dpnId, vpnName, interfaceName, lPortTag, addToDpn));
 428 //            return futures;
 429 //        }
 430 //    }
 431 //
 432 //
 433 //    /**
 434 //     * JobCallback class is used as a future callback for
 435 //     * main and rollback workers to handle success and failure.
 436 //     */
 437 //    private class UpdateDpnToVpnCallback implements FutureCallback<List<Void>> {
 438 //        BigInteger dpnId;
 439 //        String vpnName;
 440 //        String interfaceName;
 441 //        boolean addToDpn;
 442 //        int lPortTag;
 443 //
 444 //        public UpdateDpnToVpnCallback(BigInteger dpnId, String vpnName, String interfaceName,
 445 //                                      int lPortTag, boolean addToDpn) {
 446 //            this.dpnId= dpnId;
 447 //            this.vpnName = vpnName;
 448 //            this.interfaceName = interfaceName;
 449 //            this.lPortTag = lPortTag;
 450 //            this.addToDpn = addToDpn;
 451 //        }
 452 //
 453 //        /**
 454 //         * @param voids
 455 //         * This implies that all the future instances have returned success. -- TODO: Confirm this
 456 //         */
 457 //        @Override
 458 //        public void onSuccess(List<Void> voids) {
 459 //            WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
 460 //            bindService(dpnId, vpnName, interfaceName, lPortTag, writeTxn);
 461 //            processVpnInterfaceAdjacencies(dpnId, vpnName, interfaceName, writeTxn);
 462 //            writeTxn.submit();
 463 //        }
 464 //
 465 //        /**
 466 //         *
 467 //         * @param throwable
 468 //         * This method is used to handle failure callbacks.
 469 //         * If more retry needed, the retrycount is decremented and mainworker is executed again.
 470 //         * After retries completed, rollbackworker is executed.
 471 //         * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 472 //         */
 473 //
 474 //        @Override
 475 //        public void onFailure(Throwable throwable) {
 476 //            LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 477 //        }
 478 //    }
 479
 480
 481     private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
 482                                                  VpnInterface intf) {
 483         //Read NextHops
 484         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 485         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 486
 487         String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
 488         if (rd == null) {
 489             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 490                     intf.getName(), intf.getVpnInstanceName());
 491             return;
 492         } else {
 493             if (rd.equals(intf.getVpnInstanceName())) {
 494                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in " +
 495                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 496
 497                 return;
 498             }
 499         }
 500         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
 501                 intf.getVpnInstanceName(), rd);
 502
 503         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 504         if (nextHopIp == null){
 505             LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
 506             return;
 507         }
 508
 509         if (adjacencies.isPresent()) {
 510             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 511
 512             if (!nextHops.isEmpty()) {
 513                 LOG.trace("NextHops are " + nextHops);
 514                 for (Adjacency nextHop : nextHops) {
 515                     long label = nextHop.getLabel();
 516                     try {
 517                         LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 518                         bgpManager.advertisePrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
 519                         LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 520                     } catch(Exception e) {
 521                         LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
 522                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 523                     }
 524                 }
 525             }
 526         }
 527     }
 528
 529     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 530         //Read NextHops
 531         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 532         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 533
 534         String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
 535         if (rd == null) {
 536             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 537                     intf.getName(), intf.getVpnInstanceName());
 538             return;
 539         } else {
 540             if (rd.equals(intf.getVpnInstanceName())) {
 541                 LOG.info("withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in " +
 542                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 543                 return;
 544             }
 545         }
 546         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 547                 intf.getVpnInstanceName(), rd);
 548         if (adjacencies.isPresent()) {
 549             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 550
 551             if (!nextHops.isEmpty()) {
 552                 LOG.trace("NextHops are " + nextHops);
 553                 for (Adjacency nextHop : nextHops) {
 554                     try {
 555                         LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 556                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 557                         LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 558                     } catch(Exception e) {
 559                         LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
 560                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 561                     }
 562                 }
 563             }
 564         }
 565     }
 566
 567     private void updateDpnDbs(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 568         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 569         if (dpId == null) {
 570             dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, interfaceName);
 571         }
 572         if(!dpId.equals(BigInteger.ZERO)) {
 573             if(add)
 574                 updateMappingDbs(vpnId, dpId, interfaceName, vpnName);
 575             else
 576                 removeFromMappingDbs(vpnId, dpId, interfaceName, vpnName);
 577         }
 578     }
 579
 580     private void bindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName, int lPortTag,
 581                              WriteTransaction writeConfigTxn) {
 582         int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 583         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 584
 585         int instructionKey = 0;
 586         List<Instruction> instructions = new ArrayList<Instruction>();
 587
 588         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 589         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 590
 591         BoundServices
 592                 serviceInfo =
 593                 InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 594                         NwConstants.L3VPN_SERVICE_INDEX, priority,
 595                         NwConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 596         writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION,
 597                 InterfaceUtils.buildServiceId(vpnInterfaceName, NwConstants.L3VPN_SERVICE_INDEX), serviceInfo, true);
 598         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
 599                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW, writeConfigTxn);
 600         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
 601                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.ADD_FLOW, writeConfigTxn);
 602
 603     }
 604
 605     private void processVpnInterfaceAdjacencies(BigInteger dpnId, String vpnName, String interfaceName,
 606                                                 WriteTransaction writeConfigTxn,
 607                                                 WriteTransaction writeOperTxn) {
 608         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 609         // Read NextHops
 610         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 611         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
 612
 613         if (adjacencies.isPresent()) {
 614             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 615             List<Adjacency> value = new ArrayList<>();
 616
 617             // Get the rd of the vpn instance
 618             String rd = getRouteDistinguisher(vpnName);
 619
 620             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 621             if (nextHopIp == null){
 622                 LOG.error("NextHop for interface {} is null", interfaceName);
 623                 return;
 624             }
 625
 626             List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
 627
 628             LOG.trace("NextHops for interface {} are {}", interfaceName, nextHops);
 629             for (Adjacency nextHop : nextHops) {
 630                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 631                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 632                         VpnUtil.getNextHopLabelKey((rd == null) ? vpnName
 633                                 : rd, prefix));
 634                 List<String> adjNextHop = nextHop.getNextHopIpList();
 635                 value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIpList(
 636                         (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : Arrays.asList(nextHopIp))
 637                         .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 638
 639                 if (nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 640                     LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, interfaceName, vpnName);
 641                     writeOperTxn.merge(
 642                             LogicalDatastoreType.OPERATIONAL,
 643                             VpnUtil.getPrefixToInterfaceIdentifier(
 644                                     VpnUtil.getVpnId(broker, vpnName), prefix),
 645                             VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix), true);
 646                 } else {
 647                     //Extra route adjacency
 648                     LOG.trace("Adding prefix {} and nexthopList {} as extra-route for vpn", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName);
 649                     writeOperTxn.merge(
 650                             LogicalDatastoreType.OPERATIONAL,
 651                             VpnUtil.getVpnToExtrarouteIdentifier(
 652                                     (rd != null) ? rd : vpnName, nextHop.getIpAddress()),
 653                             VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()), true);
 654                 }
 655             }
 656
 657             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 658
 659             VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 660             InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 661             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, true);
 662             long vpnId = VpnUtil.getVpnId(broker, vpnName);
 663
 664             for (Adjacency nextHop : aug.getAdjacency()) {
 665                 long label = nextHop.getLabel();
 666                 List<String> nextHopList = new ArrayList<>(nextHop.getNextHopIpList());
 667                 if (rd != null) {
 668                     addToLabelMapper(label, dpnId, nextHop.getIpAddress(), nextHopList, vpnId,
 669                             interfaceName, null,false, rd, writeOperTxn);
 670                     addPrefixToBGP(rd, nextHop.getIpAddress(), nextHopIp, label, writeConfigTxn);
 671                     //TODO: ERT - check for VPNs importing my route
 672                     for (VpnInstance vpn : vpnsToImportRoute) {
 673                         String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
 674                         if (vpnRd != null) {
 675                             LOG.debug("Exporting route with rd {} prefix {} nexthop {} label {} to VPN {}", vpnRd, nextHop.getIpAddress(), nextHopIp, label, vpn);
 676                             fibManager.addOrUpdateFibEntry(broker, vpnRd, nextHop.getIpAddress(), Arrays.asList(nextHopIp), (int) label,
 677                                     RouteOrigin.SELF_IMPORTED, writeConfigTxn);
 678                         }
 679                     }
 680                 } else {
 681                     // ### add FIB route directly
 682                     fibManager.addOrUpdateFibEntry(broker, vpnName, nextHop.getIpAddress(), Arrays.asList(nextHopIp),
 683                             (int) label, RouteOrigin.STATIC, writeConfigTxn);
 684                 }
 685             }
 686         }
 687     }
 688
 689     private List<VpnInstance> getVpnsImportingMyRoute(final String vpnName) {
 690         List<VpnInstance> vpnsToImportRoute = new ArrayList<>();
 691
 692         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 693                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 694         Optional<VpnInstance> optVpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 695         final VpnInstance vpnInstance;
 696         if (optVpnInstance.isPresent()) {
 697             vpnInstance = optVpnInstance.get();
 698         } else {
 699             LOG.debug("Could not retrieve vpn instance {} to check for vpns importing the routes", vpnName);
 700             return vpnsToImportRoute;
 701         }
 702
 703         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 704             @Override
 705             public boolean apply(VpnInstance input) {
 706                 return !input.getVpnInstanceName().equals(vpnName);
 707             }
 708         };
 709
 710         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 711             @Override
 712             public boolean apply(VpnInstance input) {
 713                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ExportExtcommunity),
 714                         getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 715                 return Iterators.size(commonRTs.iterator()) > 0;
 716             }
 717         };
 718
 719         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 720             @Override
 721             public String apply(VpnInstance vpnInstance) {
 722                 //return vpnInstance.getVpnInstanceName();
 723                 return vpnInstance.getIpv4Family().getRouteDistinguisher();
 724             }
 725         };
 726
 727         vpnsToImportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(broker)).
 728                 filter(excludeVpn).
 729                 filter(matchRTs).toList();
 730         return vpnsToImportRoute;
 731     }
 732
 733     private List<VpnInstance> getVpnsExportingMyRoute(final String vpnName) {
 734         List<VpnInstance> vpnsToExportRoute = new ArrayList<>();
 735
 736         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 737                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 738         Optional<VpnInstance> optVpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 739         final VpnInstance vpnInstance;
 740         if (optVpnInstance.isPresent()) {
 741             vpnInstance = optVpnInstance.get();
 742         } else {
 743             LOG.debug("Could not retrieve vpn instance {} to check for vpns exporting the routes", vpnName);
 744             return vpnsToExportRoute;
 745         }
 746
 747         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 748             @Override
 749             public boolean apply(VpnInstance input) {
 750                 return !input.getVpnInstanceName().equals(vpnName);
 751             }
 752         };
 753
 754         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 755             @Override
 756             public boolean apply(VpnInstance input) {
 757                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ImportExtcommunity),
 758                         getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
 759                 return Iterators.size(commonRTs.iterator()) > 0;
 760             }
 761         };
 762
 763         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 764             @Override
 765             public String apply(VpnInstance vpnInstance) {
 766                 return vpnInstance.getVpnInstanceName();
 767             }
 768         };
 769
 770         vpnsToExportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(broker)).
 771                 filter(excludeVpn).
 772                 filter(matchRTs).toList();
 773         return vpnsToExportRoute;
 774     }
 775
 776     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
 777         final Predicate<T> inPredicate = Predicates.<T>in(collection2);
 778         return new Iterable<T>() {
 779             @Override
 780             public Iterator<T> iterator() {
 781                 return Iterators.filter(collection1.iterator(), inPredicate);
 782             }
 783         };
 784     }
 785
 786     private List<String> getRts(VpnInstance vpnInstance, VpnTarget.VrfRTType rtType) {
 787         String name = vpnInstance.getVpnInstanceName();
 788         List<String> rts = new ArrayList<>();
 789         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 790         if (vpnConfig == null) {
 791             LOG.trace("vpn config is not available for {}", name);
 792             return rts;
 793         }
 794         VpnTargets targets = vpnConfig.getVpnTargets();
 795         if (targets == null) {
 796             LOG.trace("vpn targets not available for {}", name);
 797             return rts;
 798         }
 799         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 800         if (vpnTargets == null) {
 801             LOG.trace("vpnTarget values not available for {}", name);
 802             return rts;
 803         }
 804         for (VpnTarget target : vpnTargets) {
 805             //TODO: Check for RT type is Both
 806             if(target.getVrfRTType().equals(rtType) ||
 807                     target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
 808                 String rtValue = target.getVrfRTValue();
 809                 rts.add(rtValue);
 810             }
 811         }
 812         return rts;
 813     }
 814
 815     private List<String> getExportRts(VpnInstance vpnInstance) {
 816         List<String> exportRts = new ArrayList<>();
 817         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 818         VpnTargets targets = vpnConfig.getVpnTargets();
 819         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 820         for (VpnTarget target : vpnTargets) {
 821             if (target.getVrfRTType().equals(VpnTarget.VrfRTType.ExportExtcommunity)) {
 822                 String rtValue = target.getVrfRTValue();
 823                 exportRts.add(rtValue);
 824             }
 825         }
 826         return exportRts;
 827     }
 828
 829     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 830                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow,
 831                              WriteTransaction writeConfigTxn){
 832         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 833         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, BigInteger.valueOf(vpnId));
 834         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 835                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 836
 837         // Matching Arp reply flows
 838         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 839         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 840                 metadata, metadataMask }));
 841
 842         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 843
 844         // Instruction to punt to controller
 845         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 846         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 847         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 848         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
 849                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
 850
 851         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 852
 853         // Install the flow entry in L3_INTERFACE_TABLE
 854         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 855                 NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 856         FlowEntity flowEntity;
 857         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 858                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 859                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 860
 861         Flow flow = flowEntity.getFlowBuilder().build();
 862         String flowId = flowEntity.getFlowId();
 863         FlowKey flowKey = new FlowKey( new FlowId(flowId));
 864         Node nodeDpn = buildDpnNode(dpId);
 865
 866         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 867                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 868                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 869
 870         if (writeConfigTxn != null) {
 871             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 872                 LOG.debug("Creating ARP Flow for interface {}", vpnInterfaceName);
 873                 writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
 874             } else {
 875                 LOG.debug("Deleting ARP Flow for interface {}", vpnInterfaceName);
 876                 writeConfigTxn.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 877             }
 878         } else {
 879             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 880                 LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 881                 mdsalManager.installFlow(flowEntity);
 882             } else {
 883                 LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 884                 mdsalManager.removeFlow(flowEntity);
 885             }
 886         }
 887     }
 888
 889     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
 890     private Node buildDpnNode(BigInteger dpnId) {
 891         NodeId nodeId = new NodeId("openflow:" + dpnId);
 892         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
 893
 894         return nodeDpn;
 895     }
 896
 897     private String getRouteDistinguisher(String vpnName) {
 898         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 899                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 900         Optional<VpnInstance> vpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 901         String rd = "";
 902         if(vpnInstance.isPresent()) {
 903             VpnInstance instance = vpnInstance.get();
 904             VpnAfConfig config = instance.getIpv4Family();
 905             rd = config.getRouteDistinguisher();
 906         }
 907         return rd;
 908     }
 909
 910     private void updateMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 911         String routeDistinguisher = getRouteDistinguisher(vpnName);
 912         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 913         synchronized (vpnName.intern()) {
 914             WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
 915             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 916             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 917             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data
 918                     .entry.vpn.to.dpn.list.VpnInterfaces
 919                     vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 920
 921             if (dpnInVpn.isPresent()) {
 922                 if (writeTxn != null) {
 923                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id.child(
 924                             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance
 925                                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 926                             new VpnInterfacesKey(intfName)), vpnInterface, true);
 927                 } else {
 928                     VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 929                             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance
 930                                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 931                             new VpnInterfacesKey(intfName)), vpnInterface);
 932                 }
 933             } else {
 934
 935                 /*
 936                 * Increment the active dpn count in VpnInstanceOpData.
 937                 * ActiveDpnCount will be decremented when all the VpnInt in that Dpn are removed.
 938                 */
 939                 InstanceIdentifier<VpnInstanceOpDataEntry> vpnInsOpDataId = VpnUtil.getVpnInstanceOpDataIdentifier(rd);
 940                 Optional<VpnInstanceOpDataEntry> vpnInstOpData = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, vpnInsOpDataId);
 941
 942                 java.lang.Long activeDpnCnt = vpnInstOpData.get().getActiveDpnCount();
 943
 944                 VpnInstanceOpDataEntryBuilder builder =
 945                         new VpnInstanceOpDataEntryBuilder(vpnInstOpData.get()).setVrfId(rd).setVpnId(vpnId).setVpnInstanceName(vpnName).setActiveDpnCount(++activeDpnCnt);
 946
 947                 if (writeTxn != null) {
 948                     writeTxn.merge(LogicalDatastoreType.OPERATIONAL,
 949                             vpnInsOpDataId,
 950                             builder.build(), true);
 951                 } else {
 952                     VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 953                             vpnInsOpDataId,
 954                             builder.build());
 955                 }
 956
 957                 VpnToDpnListBuilder vpnToDpnList = new VpnToDpnListBuilder().setDpnId(dpnId).setDpnState(VpnToDpnList.DpnState.Active);
 958                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 959                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = new ArrayList<>();
 960                 vpnInterfaces.add(vpnInterface);
 961                 if (writeTxn != null) {
 962                     writeTxn.merge(LogicalDatastoreType.OPERATIONAL, id,
 963                             vpnToDpnList.setVpnInterfaces(vpnInterfaces).build(), true);
 964                 } else {
 965                     VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, id,
 966                             vpnToDpnList.setVpnInterfaces(vpnInterfaces).build());
 967                 }
 968                 fibManager.populateFibOnNewDpn(dpnId, vpnId, (rd == null) ? vpnName : rd);
 969                 publishAddNotification(dpnId, vpnName, rd);
 970                 //TODO: IRT - import local routes to vpn. check for the VPNs exporting the routes
 971                 //FIXME: do we need to handle here. since already routes are imported to this vpn
 972
 973             }
 974             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
 975             try {
 976                 futures.get();
 977             } catch (InterruptedException | ExecutionException e) {
 978                 LOG.error("Error adding to dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
 979                 throw new RuntimeException(e.getMessage());
 980             }
 981         }
 982     }
 983
 984     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
 985         List<VpnInstance> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
 986         for (VpnInstance vpn : vpnsToExportRoute) {
 987             String rd = vpn.getIpv4Family().getRouteDistinguisher();
 988             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(broker, vpn.getIpv4Family().getRouteDistinguisher());
 989             WriteTransaction writeConfigTxn = broker.newWriteOnlyTransaction();
 990             if (vrfEntries != null) {
 991                 for (VrfEntry vrfEntry : vrfEntries) {
 992                     try {
 993                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
 994                             continue;
 995                         }
 996                         String prefix = vrfEntry.getDestPrefix();
 997                         long label = vrfEntry.getLabel();
 998                         List<String> nextHops = vrfEntry.getNextHopAddressList();
 999                         SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1000                         for (String nh : nextHops) {
1001                             if (route != null) {
1002                                 LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1003                                 importSubnetRouteForNewVpn(rd, prefix, nh, (int)label, route, writeConfigTxn);
1004                             } else {
1005                                 LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1006                                 fibManager.addOrUpdateFibEntry(broker, vpnRd, prefix, Arrays.asList(nh), (int)label,
1007                                         RouteOrigin.SELF_IMPORTED, writeConfigTxn);
1008                             }
1009                         }
1010                     } catch (Exception e) {
1011                         LOG.error("Exception occurred while importing route with prefix {} label {} nexthop {} from vpn {} to vpn {}", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpn.getVpnInstanceName(), vpnName);
1012                     }
1013                 }
1014                 writeConfigTxn.submit();
1015             } else {
1016                 LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getIpv4Family().getRouteDistinguisher());
1017             }
1018         }
1019     }
1020
1021     private void removeFromMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
1022         //TODO: Delay 'DPN' removal so that other services can cleanup the entries for this dpn
1023         synchronized (vpnName.intern()) {
1024             String rd = VpnUtil.getVpnRd(broker, vpnName);
1025             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
1026             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
1027             WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
1028             if (dpnInVpn.isPresent()) {
1029                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1030                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1031                 org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op
1032                         .data.entry.vpn.to.dpn.list.VpnInterfaces
1033                         currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1034
1035                 if (vpnInterfaces.remove(currVpnInterface)) {
1036                     if (vpnInterfaces.isEmpty()) {
1037                         List<IpAddresses> ipAddresses = dpnInVpn.get().getIpAddresses();
1038                         if (ipAddresses == null || ipAddresses.isEmpty()) {
1039
1040                            /*
1041                            * Mark the DPN state as Inactive as the Vpnintf list is empty.
1042                            * This DPN will be removed in DpnToVpnChangeListner only if its
1043                            * inactive and ActiveDpnCount is zero.
1044                            */
1045                             VpnToDpnListBuilder vpnToDpnList =
1046                                     new VpnToDpnListBuilder(dpnInVpn.get())
1047                                             .setDpnId(dpnId)
1048                                             .setDpnState(VpnToDpnList.DpnState.Inactive)
1049                                             .setVpnInterfaces(vpnInterfaces);
1050
1051                             if (writeTxn != null) {
1052                                 writeTxn.put(LogicalDatastoreType.OPERATIONAL, id , vpnToDpnList.build(), true);
1053                             } else {
1054                                 VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnList.build());
1055                             }
1056
1057                             InstanceIdentifier<VpnInstanceOpDataEntry> vpnInstOpDataId = VpnUtil.getVpnInstanceOpDataIdentifier(rd);
1058                             Optional<VpnInstanceOpDataEntry> vpnInstOpData = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, vpnInstOpDataId);
1059
1060                             java.lang.Long activeDpnCnt = vpnInstOpData.get().getActiveDpnCount();
1061
1062                             VpnInstanceOpDataEntryBuilder builder =
1063                                     new VpnInstanceOpDataEntryBuilder(vpnInstOpData.get()).setVrfId(rd).setVpnId(vpnId).setVpnInstanceName(vpnName).setActiveDpnCount(--activeDpnCnt);
1064
1065                             if (writeTxn != null) {
1066                                 writeTxn.merge(LogicalDatastoreType.OPERATIONAL, vpnInstOpDataId , builder.build(), true);
1067                             } else {
1068                                 VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, vpnInstOpDataId, builder.build());
1069                             }
1070
1071                             LOG.debug("Sending cleanup event for dpn {} in VPN {}", dpnId, vpnName);
1072                             fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
1073                             publishRemoveNotification(dpnId, vpnName, rd);
1074                         } else {
1075                             LOG.debug("vpn interfaces are empty but ip addresses are present for the vpn {} in dpn {}", vpnName, dpnId);
1076                         }
1077                     } else {
1078                         if (writeTxn != null) {
1079                             writeTxn.delete(LogicalDatastoreType.OPERATIONAL, id.child(
1080                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1081                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1082                                     new VpnInterfacesKey(intfName)));
1083                         } else {
1084                             VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
1085                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1086                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1087                                     new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
1088                         }
1089                     }
1090                 }
1091             }
1092             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
1093             try {
1094                 futures.get();
1095             } catch (InterruptedException | ExecutionException e) {
1096                 LOG.error("Error removing from dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
1097                 throw new RuntimeException(e.getMessage());
1098             }
1099         }
1100     }
1101
1102     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label, WriteTransaction writeConfigTxn) {
1103         try {
1104             LOG.info("ADD: Adding Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1105             fibManager.addOrUpdateFibEntry(broker, rd, prefix, Arrays.asList(nextHopIp), (int)label, RouteOrigin.STATIC, writeConfigTxn);
1106             bgpManager.advertisePrefix(rd, prefix, Arrays.asList(nextHopIp), (int)label);
1107             LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1108         } catch(Exception e) {
1109             LOG.error("Add prefix failed", e);
1110         }
1111     }
1112
1113
1114     private InstanceIdentifier<VpnInterface> getWildCardPath() {
1115         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
1116     }
1117
1118     @Override
1119     public void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1120         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
1121         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1122         final String interfaceName = key.getName();
1123
1124         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1125         final Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1126         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
1127                 InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
1128         if (optVpnInterface.isPresent()){
1129             BigInteger dpnId = BigInteger.ZERO;
1130             Boolean dpnIdRetrieved = Boolean.FALSE;
1131             if(interfaceState != null){
1132                 try{
1133                     dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1134                     dpnIdRetrieved = Boolean.TRUE;
1135                 }catch (Exception e){
1136                     LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", interfaceName, e);
1137                 }
1138             } else {
1139                 LOG.error("Unable to retrieve interfaceState for interface {} , quitting ", interfaceName);
1140                 return;
1141             }
1142             final VpnInterface vpnOpInterface = optVpnInterface.get();
1143             if(dpnIdRetrieved == Boolean.FALSE){
1144                 LOG.info("dpnId for {} has not been retrieved yet. Fetching from vpn interface operational DS", interfaceName);
1145                 dpnId = vpnOpInterface.getDpnId();
1146             }
1147             final int ifIndex = interfaceState.getIfIndex();
1148             final BigInteger dpId = dpnId;
1149             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1150             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1151                     new Callable<List<ListenableFuture<Void>>>() {
1152                         @Override
1153                         public List<ListenableFuture<Void>> call() throws Exception {
1154                             WriteTransaction writeConfigTxn = broker.newWriteOnlyTransaction();
1155                             WriteTransaction writeOperTxn = broker.newWriteOnlyTransaction();
1156
1157                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, false, true, writeConfigTxn, writeOperTxn);
1158                             CheckedFuture<Void, TransactionCommitFailedException> futures = writeOperTxn.submit();
1159                             try {
1160                                 futures.get();
1161                             } catch (InterruptedException | ExecutionException e) {
1162                                 LOG.error("Error removing interface {} from vpn {} on dpn {}", interfaceName,
1163                                         vpnOpInterface.getVpnInstanceName(), dpId);
1164                                 throw new RuntimeException(e.getMessage());
1165                             }
1166                             futures = writeConfigTxn.submit();
1167                             try {
1168                                 futures.get();
1169                             } catch (InterruptedException | ExecutionException e) {
1170                                 LOG.error("Error removing interface {} from vpn {} on dpn {}", interfaceName,
1171                                         vpnOpInterface.getVpnInstanceName(), dpId);
1172                                 throw new RuntimeException(e.getMessage());
1173                             }
1174                             return null;
1175                         }
1176                     });
1177
1178         }else{
1179             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
1180         }
1181     }
1182
1183     protected void processVpnInterfaceDown(BigInteger dpId, String interfaceName, int lPortTag, boolean isInterfaceStateDown,
1184                                            boolean isConfigRemoval, WriteTransaction writeConfigTxn, WriteTransaction writeOperTxn) {
1185         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1186         if (!isInterfaceStateDown) {
1187             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
1188             if(vpnInterface == null){
1189                 LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
1190                 return;
1191             }else{
1192                 final String vpnName = vpnInterface.getVpnInstanceName();
1193                 if(!vpnInterface.isScheduledForRemove()){
1194                     VpnUtil.scheduleVpnInterfaceForRemoval(broker, interfaceName, dpId, vpnName, Boolean.TRUE, writeOperTxn);
1195                     removeAdjacenciesFromVpn(dpId, interfaceName, vpnInterface.getVpnInstanceName(), writeConfigTxn);
1196                     LOG.info("Unbinding vpn service from interface {} ", interfaceName);
1197                     unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown, isConfigRemoval, writeConfigTxn);
1198                 }else{
1199                     LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ", interfaceName);
1200                     return;
1201                 }
1202             }
1203         } else {
1204             // Interface is retained in the DPN, but its Link Down.
1205             // Only withdraw the prefixes for this interface from BGP
1206             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
1207             if(vpnInterface == null){
1208                 LOG.info("Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in operational data store", interfaceName);
1209                 return;
1210             }else {
1211                 withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
1212             }
1213         }
1214     }
1215
1216     private void waitForFibToRemoveVpnPrefix(String interfaceName) {
1217         // FIB didn't get a chance yet to clean up this VPNInterface
1218         // Let us give it a chance here !
1219         LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
1220         try {
1221             Runnable notifyTask = new VpnNotifyTask();
1222             vpnIntfMap.put(interfaceName, notifyTask);
1223             synchronized (notifyTask) {
1224                 try {
1225                     notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
1226                 } catch (InterruptedException e) {
1227                 }
1228             }
1229         } finally {
1230             vpnIntfMap.remove(interfaceName);
1231         }
1232     }
1233
1234     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final String interfaceName, final String vpnName,
1235                                           WriteTransaction writeConfigTxn) {
1236         //Read NextHops
1237         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1238         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1239         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
1240
1241         String rd = VpnUtil.getVpnRd(broker, vpnName);
1242         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", interfaceName,
1243                 vpnName, rd);
1244         if (adjacencies.isPresent()) {
1245             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1246
1247             if (!nextHops.isEmpty()) {
1248                 LOG.trace("NextHops are " + nextHops);
1249                 for (Adjacency nextHop : nextHops) {
1250                     List<String> nhList = new ArrayList<String>();
1251                     if (nextHop.getMacAddress() == null || nextHop.getMacAddress().isEmpty()) {
1252                         // This is either an extra-route (or) a learned IP via subnet-route
1253                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
1254                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1255                             LOG.error("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
1256                                     rd, nextHop.getIpAddress());
1257                             continue;
1258                         }
1259                         nhList = Arrays.asList(nextHopIp);
1260                     } else {
1261                         // This is a primary adjacency
1262                         nhList = nextHop.getNextHopIpList();
1263                     }
1264                     if (rd.equals(vpnName)) {
1265                         //this is an internal vpn - the rd is assigned to the vpn instance name;
1266                         //remove from FIB directly
1267                         for(String nh : nhList) {
1268                             fibManager.removeOrUpdateFibEntry(broker, vpnName, nextHop.getIpAddress(), nh, writeConfigTxn);
1269                         }
1270                     } else {
1271                         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1272                         for (String nh : nextHop.getNextHopIpList()) {
1273                             //IRT: remove routes from other vpns importing it
1274                             removePrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1275                             for (VpnInstance vpn : vpnsToImportRoute) {
1276                                 String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
1277                                 if (vpnRd != null) {
1278                                     LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd, nextHop.getIpAddress(), vpn.getVpnInstanceName());
1279                                     fibManager.removeOrUpdateFibEntry(broker, vpnRd, nextHop.getIpAddress(), nh, writeConfigTxn);
1280                                 }
1281                             }
1282                         }
1283                     }
1284                     String ip = nextHop.getIpAddress().split("/")[0];
1285                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(broker,
1286                             vpnName, ip);
1287                     if (vpnPortipToPort != null && !vpnPortipToPort.isConfig()) {
1288                         LOG.trace("VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
1289                                 vpnPortipToPort.getPortName(),ip);
1290                         VpnUtil.removeVpnPortFixedIpToPort(broker, vpnName, ip);
1291                     }
1292                 }
1293             }
1294         }
1295     }
1296
1297
1298     private void unbindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName,
1299                                int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval,
1300                                WriteTransaction writeConfigTxn) {
1301         if (!isInterfaceStateDown && isConfigRemoval) {
1302             writeConfigTxn.delete(LogicalDatastoreType.CONFIGURATION,
1303                     InterfaceUtils.buildServiceId(vpnInterfaceName,
1304                             NwConstants.L3VPN_SERVICE_INDEX));
1305         }
1306         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
1307         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
1308                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW, writeConfigTxn);
1309         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
1310                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.DEL_FLOW, writeConfigTxn);
1311     }
1312
1313
1314     private void removePrefixFromBGP(String rd, String prefix, String nextHop, WriteTransaction writeConfigTxn) {
1315         try {
1316             LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
1317             fibManager.removeOrUpdateFibEntry(broker, rd, prefix, nextHop, writeConfigTxn);
1318             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1319             LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
1320         } catch(Exception e) {
1321             LOG.error("Delete prefix failed", e);
1322         }
1323     }
1324
1325     @Override
1326     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
1327         LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
1328         String oldVpnName = original.getVpnInstanceName();
1329         String newVpnName = update.getVpnInstanceName();
1330         BigInteger dpnId = update.getDpnId();
1331         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
1332         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
1333         if (oldAdjs == null) {
1334             oldAdjs = new ArrayList<>();
1335         }
1336         if (newAdjs == null) {
1337             newAdjs = new ArrayList<>();
1338         }
1339         //handles switching between <internal VPN - external VPN>
1340         if (!oldVpnName.equals(newVpnName)) {
1341             remove(identifier, original);
1342             waitForFibToRemoveVpnPrefix(update.getName());
1343             add(identifier, update);
1344         }
1345         //handle both addition and removal of adjacencies
1346         //currently, new adjacency may be an extra route
1347         if (!oldAdjs.equals(newAdjs)) {
1348             for (Adjacency adj : newAdjs) {
1349                 if (oldAdjs.contains(adj)) {
1350                     oldAdjs.remove(adj);
1351                 } else {
1352                     // add new adjacency - right now only extra route will hit this path
1353                     addNewAdjToVpnInterface(identifier, adj, dpnId);
1354                 }
1355             }
1356             for (Adjacency adj : oldAdjs) {
1357                 delAdjFromVpnInterface(identifier, adj, dpnId);
1358             }
1359         }
1360     }
1361
1362     public void processArpRequest(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715
1363                                           .IpAddress srcIP, PhysAddress srcMac, org.opendaylight.yang.gen.v1.urn.ietf
1364             .params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress targetIP, PhysAddress targetMac, String srcInterface){
1365         //Build ARP response with ARP requests TargetIp TargetMac as the Arp Response SrcIp and SrcMac
1366         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
1367                 .setDstIpaddress(srcIP).setDstMacaddress(srcMac).setSrcIpaddress(targetIP).setSrcMacaddress(targetMac).build();
1368         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
1369         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
1370         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
1371             @Override
1372             public void onFailure(Throwable error) {
1373                 LOG.error("Error - {}", msgFormat, error);
1374             }
1375
1376             @Override
1377             public void onSuccess(RpcResult<Void> result) {
1378                 if(!result.isSuccessful()) {
1379                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
1380                 } else {
1381                     LOG.debug("Successful RPC Result - {}", msgFormat);
1382                 }
1383             }
1384         });
1385     }
1386
1387     private String getErrorText(Collection<RpcError> errors) {
1388         StringBuilder errorText = new StringBuilder();
1389         for(RpcError error : errors) {
1390             errorText.append(",").append(error.getErrorType()).append("-")
1391                     .append(error.getMessage());
1392         }
1393         return errorText.toString();
1394     }
1395
1396     private void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1397                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd,
1398                                   WriteTransaction writeOperTxn) {
1399         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1400         Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
1401         Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
1402         Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
1403         if (!isSubnetRoute) {
1404             // NextHop must be present for non-subnetroute entries
1405             Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
1406         }
1407         LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
1408         if (dpnId != null) {
1409             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1410                     .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
1411             LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1412             lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(vpnId)
1413                     .setIsSubnetRoute(isSubnetRoute);
1414             if (elanTag != null) {
1415                 lriBuilder.setElanTag(elanTag);
1416             }
1417             if (vpnInterfaceName != null) {
1418                 lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1419             }
1420             lriBuilder.setParentVpnRd(rd);
1421             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(broker, rd);
1422             if (vpnInstanceOpDataEntry != null) {
1423                 List<String> vpnInstanceNames = Arrays.asList(vpnInstanceOpDataEntry.getVpnInstanceName());
1424                 lriBuilder.setVpnInstanceList(vpnInstanceNames);
1425             }
1426             LabelRouteInfo lri = lriBuilder.build();
1427             LOG.trace("Adding route info to label map: {}", lri);
1428             if (writeOperTxn != null) {
1429                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, lriIid, lri, true);
1430             } else {
1431                 VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
1432             }
1433         } else {
1434             LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
1435         }
1436     }
1437
1438     public synchronized void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
1439                                                         long elantag, BigInteger dpnId, WriteTransaction writeTxn) {
1440         SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
1441         RouteOrigin origin = RouteOrigin.STATIC; // Only case when a route is considered as directly connected
1442         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1443                 .setLabel((long)label).setOrigin(origin.getValue())
1444                 .addAugmentation(SubnetRoute.class, route).build();
1445
1446         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
1447
1448         //TODO: What should be parentVpnId? Get it from RD?
1449         long vpnId = VpnUtil.getVpnId(broker, vpnName);
1450         addToLabelMapper((long)label, dpnId, prefix, Arrays.asList(nextHop), vpnId, null, elantag, true, rd, null);
1451         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1452
1453         InstanceIdentifierBuilder<VrfTables> idBuilder =
1454                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1455         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1456
1457         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1458                 setVrfEntry(vrfEntryList).build();
1459
1460         if (writeTxn != null) {
1461             writeTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1462         } else {
1463             VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1464         }
1465
1466         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1467         if (vpnsToImportRoute.size() > 0) {
1468             VrfEntry importingVrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1469                     .setLabel((long) label).setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
1470                     .addAugmentation(SubnetRoute.class, route).build();
1471             List<VrfEntry> importingVrfEntryList = Arrays.asList(importingVrfEntry);
1472             for (VpnInstance vpnInstance : vpnsToImportRoute) {
1473                 LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop, label, vpnInstance.getVpnInstanceName());
1474                 String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1475                 InstanceIdentifier<VrfTables> importingVrfTableId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(importingRd)).build();
1476                 VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(importingVrfEntryList).build();
1477                 if (writeTxn != null) {
1478                     writeTxn.merge(LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable, true);
1479                 } else {
1480                     VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable);
1481                 }
1482             }
1483         }
1484     }
1485
1486     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1487                                                         SubnetRoute route, WriteTransaction writeConfigTxn) {
1488
1489         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1490         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1491                 .setLabel((long)label).setOrigin(origin.getValue())
1492                 .addAugmentation(SubnetRoute.class, route).build();
1493         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, route.getElantag());
1494         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1495         InstanceIdentifierBuilder<VrfTables> idBuilder =
1496                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1497         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1498         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1499                 setVrfEntry(vrfEntryList).build();
1500         if (writeConfigTxn != null) {
1501             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1502         } else {
1503             VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1504         }
1505     }
1506
1507     public synchronized void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName){
1508         fibManager.removeOrUpdateFibEntry(broker, rd, prefix, null /* nextHopToRemove */, null);
1509         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1510         for (VpnInstance vpnInstance : vpnsToImportRoute) {
1511             String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1512             LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix, vpnInstance.getVpnInstanceName());
1513             fibManager.removeOrUpdateFibEntry(broker, importingRd, prefix, null /* nextHopToRemove */, null);
1514         }
1515     }
1516
1517     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1518
1519         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
1520
1521         if (optVpnInterface.isPresent()) {
1522             VpnInterface currVpnIntf = optVpnInterface.get();
1523             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1524             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1525             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1526             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, adjPath);
1527             long label =
1528                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1529                             VpnUtil.getNextHopLabelKey((rd != null) ? rd : currVpnIntf.getVpnInstanceName(), prefix));
1530
1531             List<Adjacency> adjacencies;
1532             if (optAdjacencies.isPresent()) {
1533                 adjacencies = optAdjacencies.get().getAdjacency();
1534             } else {
1535                 //This code will not be hit since VM adjacency will always be there
1536                 adjacencies = new ArrayList<>();
1537             }
1538
1539             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIpList(adj.getNextHopIpList())
1540                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
1541
1542             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1543             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove());
1544
1545             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1546             for (String nh : adj.getNextHopIpList()) {
1547                 addExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(), (int) label,
1548                         currVpnIntf.getName());
1549             }
1550         }
1551     }
1552
1553     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1554         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
1555
1556         if (optVpnInterface.isPresent()) {
1557             VpnInterface currVpnIntf = optVpnInterface.get();
1558
1559             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1560             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
1561             if (optAdjacencies.isPresent()) {
1562                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1563
1564                 if (!adjacencies.isEmpty()) {
1565                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1566                     LOG.trace("Adjacencies are " + adjacencies);
1567                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1568                     while (adjIt.hasNext()) {
1569                         Adjacency adjElem = adjIt.next();
1570                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1571                             adjIt.remove();
1572
1573                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1574                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1575                                     currVpnIntf.getVpnInstanceName(),
1576                                     aug, dpnId, currVpnIntf.isScheduledForRemove());
1577
1578                             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1579
1580                             for (String nh : adj.getNextHopIpList()) {
1581                                 delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
1582                                         currVpnIntf.getName());
1583                             }
1584                             break;
1585                         }
1586
1587                     }
1588                 }
1589             }
1590         }
1591
1592     }
1593
1594     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label,
1595                                  String intfName) {
1596
1597         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1598         VpnUtil.syncUpdate(
1599                 broker,
1600                 LogicalDatastoreType.OPERATIONAL,
1601                 VpnUtil.getVpnToExtrarouteIdentifier( (rd != null) ? rd : routerID, destination),
1602                 VpnUtil.getVpnToExtraroute(destination, Arrays.asList(nextHop)));
1603
1604         BigInteger dpnId = null;
1605         if (intfName != null && !intfName.isEmpty()) {
1606             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1607             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
1608             if (nextHopIp == null || nextHopIp.isEmpty()) {
1609                 LOG.error("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1610                         intfName, destination);
1611                 return;
1612             }
1613             nextHop = nextHopIp;
1614         }
1615         List<String> nextHopIpList = Arrays.asList(nextHop);
1616         addToLabelMapper((long)label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(broker, routerID),
1617                 intfName, null, false, rd, null);
1618
1619         // TODO (eperefr): This is a limitation to be stated in docs. When configuring static route to go to
1620         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1621         // first place.
1622         InterVpnLink interVpnLink = VpnUtil.getInterVpnLinkByEndpointIp(broker, nextHop);
1623         if ( interVpnLink != null ) {
1624             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1625             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1626             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1627             // This is like leaking one of the Vpn2 routes towards Vpn1
1628             boolean nexthopIsVpn2 = ( interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nextHop) );
1629             String srcVpnUuid = (nexthopIsVpn2) ? interVpnLink.getSecondEndpoint().getVpnUuid().getValue()
1630                     : interVpnLink.getFirstEndpoint().getVpnUuid().getValue();
1631             String dstVpnUuid = (nexthopIsVpn2) ? interVpnLink.getFirstEndpoint().getVpnUuid().getValue()
1632                     : interVpnLink.getSecondEndpoint().getVpnUuid().getValue();
1633             String dstVpnRd = VpnUtil.getVpnRd(broker, dstVpnUuid);
1634             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1635                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1636             VpnUtil.leakRoute(broker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel);
1637         } else {
1638             if (rd != null) {
1639                 addPrefixToBGP(rd, destination, nextHop, label, null);
1640             } else {
1641                 // ### add FIB route directly
1642                 fibManager.addOrUpdateFibEntry(broker, routerID, destination, Arrays.asList(nextHop), label, RouteOrigin.STATIC, null);
1643             }
1644         }
1645     }
1646
1647     protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName) {
1648         if (intfName != null && !intfName.isEmpty()) {
1649             BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1650             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
1651             if (nextHopIp == null || nextHopIp.isEmpty()) {
1652                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1653                         intfName, destination);
1654             }
1655             nextHop = nextHopIp;
1656         }
1657
1658         if (rd != null) {
1659             removePrefixFromBGP(rd, destination, nextHop, null);
1660         } else {
1661             // ### add FIB route directly
1662             fibManager.removeOrUpdateFibEntry(broker, routerID, destination, nextHop, null);
1663         }
1664     }
1665
1666     class VpnInterfaceOpListener extends org.opendaylight.genius.mdsalutil.AbstractDataChangeListener<VpnInterface> {
1667
1668         public VpnInterfaceOpListener() {
1669             super(VpnInterface.class);
1670         }
1671
1672         @Override
1673         protected void remove(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface del) {
1674             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1675             final String interfaceName = key.getName();
1676             final String vpnName = del.getVpnInstanceName();
1677             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1678             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1679                     new Callable<List<ListenableFuture<Void>>>() {
1680                         @Override
1681                         public List<ListenableFuture<Void>> call() throws Exception {
1682                             WriteTransaction writeOperTxn = broker.newWriteOnlyTransaction();
1683                             postProcessVpnInterfaceRemoval(identifier, del, writeOperTxn);
1684                             CheckedFuture<Void, TransactionCommitFailedException> futures = writeOperTxn.submit();
1685                             try {
1686                                 futures.get();
1687                             } catch (InterruptedException | ExecutionException e) {
1688                                 LOG.error("Error handling removal of oper data for interface {} from vpn {}", interfaceName,
1689                                         vpnName);
1690                                 throw new RuntimeException(e.getMessage());
1691                             }
1692                             return null;
1693                         }
1694                     });
1695         }
1696
1697         private void postProcessVpnInterfaceRemoval(InstanceIdentifier<VpnInterface> identifier, VpnInterface del,
1698                                                     WriteTransaction writeOperTxn) {
1699             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1700             String interfaceName = key.getName();
1701             String vpnName = del.getVpnInstanceName();
1702
1703             LOG.trace("VpnInterfaceOpListener removed: interface name {} vpnName {}", interfaceName, vpnName);
1704             //decrement the vpn interface count in Vpn Instance Op Data
1705             InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to
1706                     .vpn.id.VpnInstance>
1707                     id = VpnUtil.getVpnInstanceToVpnIdIdentifier(vpnName);
1708             Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id
1709                     .VpnInstance> vpnInstance
1710                     = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
1711
1712             if (vpnInstance.isPresent()) {
1713                 String rd = null;
1714                 rd = vpnInstance.get().getVrfId();
1715                 //String rd = getRouteDistinguisher(del.getVpnInstanceName());
1716
1717                 VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
1718                 LOG.trace("VpnInterfaceOpListener removed: interface name {} rd {} vpnName {} in Vpn Op Instance {}",
1719                         interfaceName, rd, vpnName, vpnInstOp);
1720
1721                 if (vpnInstOp != null) {
1722                     // Vpn Interface removed => No more adjacencies from it.
1723                     // Hence clean up interface from vpn-dpn-interface list.
1724                     Adjacency adjacency = del.getAugmentation(Adjacencies.class).getAdjacency().get(0);
1725                     List<Prefixes> prefixToInterface = new ArrayList<>();
1726                     Optional<Prefixes> prefix = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1727                             VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1728                                     VpnUtil.getIpPrefix(adjacency.getIpAddress())));
1729                     if (prefix.isPresent()) {
1730                         prefixToInterface.add(prefix.get());
1731                     }
1732                     if (prefixToInterface.isEmpty()) {
1733                         for (String nh : adjacency.getNextHopIpList()) {
1734                             prefix = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1735                                     VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1736                                             VpnUtil.getIpPrefix(nh)));
1737                             if (prefix.isPresent())
1738                                 prefixToInterface.add(prefix.get());
1739                         }
1740                     }
1741                     for (Prefixes pref : prefixToInterface) {
1742                         if (writeOperTxn != null) {
1743                             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1744                                     VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(), pref.getIpAddress()));
1745                         } else {
1746                             VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
1747                                     VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(), pref.getIpAddress()),
1748                                     VpnUtil.DEFAULT_CALLBACK);
1749                         }
1750                         updateDpnDbs(pref.getDpnId(), del.getVpnInstanceName(), interfaceName, false /* delete */);
1751                     }
1752                 }
1753             } else {
1754                 LOG.error("rd not retrievable as vpninstancetovpnid for vpn {} is absent, trying rd as ", vpnName, vpnName);
1755             }
1756             notifyTaskIfRequired(interfaceName);
1757         }
1758
1759         private void notifyTaskIfRequired(String intfName) {
1760             Runnable notifyTask = vpnIntfMap.remove(intfName);
1761             if (notifyTask == null) {
1762                 LOG.trace("VpnInterfaceOpListener update: No Notify Task queued for vpnInterface {}", intfName);
1763                 return;
1764             }
1765             executorService.execute(notifyTask);
1766         }
1767
1768         @Override
1769         protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1770                               final VpnInterface update) {
1771             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1772             final String interfaceName = key.getName();
1773
1774             if (original.getVpnInstanceName().equals(update.getVpnInstanceName())) {
1775                 return;
1776             }
1777
1778             final String vpnName = update.getVpnInstanceName();
1779             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1780             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1781                     new Callable<List<ListenableFuture<Void>>>() {
1782                         @Override
1783                         public List<ListenableFuture<Void>> call() throws Exception {
1784                             postProcessVpnInterfaceUpdate(identifier, original, update);
1785                             return null;
1786                         }
1787                     });
1788         }
1789
1790         private void postProcessVpnInterfaceUpdate(InstanceIdentifier<VpnInterface> identifier, VpnInterface original,
1791                                                    VpnInterface update) {
1792             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1793             String interfaceName = key.getName();
1794
1795             //increment the vpn interface count in Vpn Instance Op Data
1796             VpnInstanceOpDataEntry vpnInstOp = null;
1797             InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to
1798                     .vpn.id.VpnInstance>
1799                     origId = VpnUtil.getVpnInstanceToVpnIdIdentifier(original.getVpnInstanceName());
1800             Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id
1801                     .VpnInstance> origVpnInstance
1802                     = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, origId);
1803
1804             if (origVpnInstance.isPresent()) {
1805                 String rd = null;
1806                 rd = origVpnInstance.get().getVrfId();
1807
1808                 vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
1809                 LOG.trace("VpnInterfaceOpListener updated: interface name {} original rd {} original vpnName {}",
1810                         interfaceName, rd, original.getVpnInstanceName());
1811
1812                 if (vpnInstOp != null) {
1813                     Adjacency adjacency = original.getAugmentation(Adjacencies.class).getAdjacency().get(0);
1814                     List<Prefixes> prefixToInterfaceList = new ArrayList<>();
1815                     Optional<Prefixes> prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1816                             VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1817                                     VpnUtil.getIpPrefix(adjacency.getIpAddress())));
1818                     if (prefixToInterface.isPresent()) {
1819                         prefixToInterfaceList.add(prefixToInterface.get());
1820                     } else {
1821                         for (String adj : adjacency.getNextHopIpList()) {
1822                             prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1823                                     VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1824                                             VpnUtil.getIpPrefix(adj)));
1825                             if (prefixToInterface.isPresent()) {
1826                                 prefixToInterfaceList.add(prefixToInterface.get());
1827                             }
1828                         }
1829                     }
1830                     for (Prefixes prefix : prefixToInterfaceList) {
1831                         updateDpnDbs(prefix.getDpnId(), original.getVpnInstanceName(), interfaceName, false /* delete */);
1832                     }
1833                 }
1834             }
1835             notifyTaskIfRequired(interfaceName);
1836         }
1837
1838         @Override
1839         protected void add(InstanceIdentifier<VpnInterface> identifier, VpnInterface add) {
1840         }
1841     }
1842
1843     void publishAddNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1844         LOG.debug("Sending notification for add dpn {} in vpn {} event ", dpnId, vpnName);
1845         AddEventData data = new AddEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1846         AddDpnEvent event = new AddDpnEventBuilder().setAddEventData(data).build();
1847         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1848         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1849             @Override
1850             public void onFailure(Throwable error) {
1851                 LOG.warn("Error in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName, error);
1852             }
1853
1854             @Override
1855             public void onSuccess(Object arg) {
1856                 LOG.trace("Successful in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName);
1857             }
1858         });
1859     }
1860
1861     void publishRemoveNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1862         LOG.debug("Sending notification for remove dpn {} in vpn {} event ", dpnId, vpnName);
1863         RemoveEventData data = new RemoveEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1864         RemoveDpnEvent event = new RemoveDpnEventBuilder().setRemoveEventData(data).build();
1865         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1866         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1867             @Override
1868             public void onFailure(Throwable error) {
1869                 LOG.warn("Error in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName, error);
1870             }
1871
1872             @Override
1873             public void onSuccess(Object arg) {
1874                 LOG.trace("Successful in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName);
1875             }
1876         });
1877     }
1878
1879     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1880         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1881                 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1882                 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1883     }
1884
1885     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1886         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1887                 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1888     }
1889
1890     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1891         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1892         if(dpId.equals(BigInteger.ZERO)) {
1893             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
1894             return;
1895         }
1896         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1897
1898         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
1899                 .OPERATIONAL, routerDpnListIdentifier);
1900         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1901         if (optionalRouterDpnList.isPresent()) {
1902             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1903                     RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1904         } else {
1905             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1906             builder.setRouterId(routerName);
1907             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1908             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
1909             routerInterfaces.add(routerInterface);
1910             builder.setDpnVpninterfacesList(Arrays.asList(dpnVpnList.build()));
1911             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1912                     getRouterId(routerName),
1913                     builder.build(), true);
1914         }
1915     }
1916
1917     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1918         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1919         if(dpId.equals(BigInteger.ZERO)) {
1920             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1921             return;
1922         }
1923         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1924         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
1925                 .OPERATIONAL, routerDpnListIdentifier);
1926         if (optionalRouterDpnList.isPresent()) {
1927             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1928             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1929
1930             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1931                 if (routerInterfaces.isEmpty()) {
1932                     if (writeOperTxn != null) {
1933                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1934                     } else {
1935                         MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1936                     }
1937                 } else {
1938                     if (writeOperTxn != null) {
1939                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1940                                 RouterInterfaces.class,
1941                                 new RouterInterfacesKey(vpnInterfaceName)));
1942                     } else {
1943                         MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1944                                 RouterInterfaces.class,
1945                                 new RouterInterfacesKey(vpnInterfaceName)));
1946                     }
1947                 }
1948             }
1949         }
1950     }
1951
1952     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1953                                                   WriteTransaction writeOperTxn) {
1954         if(dpId.equals(BigInteger.ZERO)) {
1955             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1956             return;
1957         }
1958         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1959         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
1960                 .OPERATIONAL, routerDpnListIdentifier);
1961         if (optionalRouterDpnList.isPresent()) {
1962             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1963             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1964             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1965                 if (routerInterfaces.isEmpty()) {
1966                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1967                 } else {
1968                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1969                             RouterInterfaces.class,
1970                             new RouterInterfacesKey(vpnInterfaceName)));
1971                 }
1972             }
1973         }
1974     }
1975
1976     public void addMIPAdjacency(String vpnName,String vpnInterface, org.opendaylight.yang.gen.v1.urn.ietf.params.xml
1977             .ns.yang.ietf.inet.types.rev130715.IpAddress prefix){
1978
1979         LOG.trace("Adding {} adjacency to VPN Interface {} ",prefix,vpnInterface);
1980         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
1981         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
1982         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
1983         String nextHopIpAddr = null;
1984         String nextHopMacAddress = null;
1985         String ip = prefix.getIpv4Address().getValue();
1986         if (adjacencies.isPresent()) {
1987             List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
1988             ip = VpnUtil.getIpPrefix(ip);
1989             for (Adjacency adjacs : adjacencyList) {
1990                 if (adjacs.getMacAddress() != null && !adjacs.getMacAddress().isEmpty()) {
1991                     nextHopIpAddr = adjacs.getIpAddress();
1992                     nextHopMacAddress = adjacs.getMacAddress();
1993                     break;
1994                 }
1995             }
1996             if (nextHopMacAddress != null && ip != null) {
1997                 synchronized (vpnInterface.intern()) {
1998                     String rd = VpnUtil.getVpnRd(broker, vpnName);
1999                     long label =
2000                             VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
2001                                     VpnUtil.getNextHopLabelKey((rd != null) ? rd : vpnName, ip));
2002                     String nextHopIp = nextHopIpAddr.split("/")[0];
2003                     Adjacency newAdj = new AdjacencyBuilder().setIpAddress(ip).setKey
2004                             (new AdjacencyKey(ip)).setNextHopIpList(Arrays.asList(nextHopIp)).build();
2005                     adjacencyList.add(newAdj);
2006                     Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
2007                     VpnInterface newVpnIntf = new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(vpnInterface)).
2008                             setName(vpnInterface).setVpnInstanceName(vpnName).addAugmentation(Adjacencies.class, aug).build();
2009                     VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vpnIfId, newVpnIntf);
2010                 }
2011                 LOG.debug(" Successfully stored subnetroute Adjacency into VpnInterface {}", vpnInterface);
2012             }
2013         }
2014
2015     }
2016
2017     public void removeMIPAdjacency(String vpnName, String vpnInterface, org.opendaylight.yang.gen.v1.urn.ietf.params
2018             .xml.ns.yang.ietf.inet.types.rev130715.IpAddress prefix) {
2019         String ip = VpnUtil.getIpPrefix(prefix.getIpv4Address().getValue());
2020         LOG.trace("Removing {} adjacency from Old VPN Interface {} ",ip,vpnInterface);
2021         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
2022         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
2023         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
2024         if (adjacencies.isPresent()) {
2025             synchronized (vpnInterface.intern()) {
2026                 InstanceIdentifier<Adjacency> adjacencyIdentifier = InstanceIdentifier.builder(VpnInterfaces.class).
2027                         child(VpnInterface.class, new VpnInterfaceKey(vpnInterface)).augmentation(Adjacencies.class)
2028                         .child(Adjacency.class, new AdjacencyKey(ip)).build();
2029                 MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, adjacencyIdentifier);
2030             }
2031             LOG.trace("Successfully Deleted Adjacency into VpnInterface {}", vpnInterface);
2032         }
2033     }
2034 }