vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.*;
  11 import com.google.common.collect.FluentIterable;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.*;
  14 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  15 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  16 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  17 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  18 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  19 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  20
  21 import org.opendaylight.controller.md.sal.binding.api.*;
  22 import org.opendaylight.genius.mdsalutil.*;
  23 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  24 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.VpnTargets;
  25 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  26 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeExternal;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeHwvtep;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeInternal;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.IsDcgwPresentInputBuilder;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.IsDcgwPresentOutput;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.*;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventData;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventDataBuilder;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventData;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventDataBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  72
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
  74         .VpnInstanceOpDataEntryBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInput;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInputBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  91
  92 import java.math.BigInteger;
  93 import java.util.Collection;
  94 import java.util.List;
  95 import java.util.ArrayList;
  96 import java.util.Arrays;
  97 import java.util.Iterator;
  98 import java.util.concurrent.*;
  99
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 103 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 107 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 108 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 109 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 110 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 111 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
 112 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
 116 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
 117 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
 118 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
 119 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
 120 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
 121 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
 122 import org.opendaylight.yangtools.yang.common.RpcError;
 123 import org.opendaylight.yangtools.yang.common.RpcResult;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
 128     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 129     private ListenerRegistration<DataChangeListener> listenerRegistration, opListenerRegistration;
 130     private final DataBroker broker;
 131     private final IBgpManager bgpManager;
 132     private IFibManager fibManager;
 133     private IMdsalApiManager mdsalManager;
 134     private OdlInterfaceRpcService ifaceMgrRpcService;
 135     private IdManagerService idManager;
 136     private OdlArputilService arpManager;
 137     private NeutronvpnService neuService;
 138     private VpnSubnetRouteHandler vpnSubnetRouteHandler;
 139     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
 140     private ExecutorService executorService = Executors.newSingleThreadExecutor();
 141     private InterfaceStateChangeListener interfaceListener;
 142     private SubnetRouteInterfaceStateChangeListener subnetRouteInterfaceListener;
 143     private TunnelInterfaceStateListener tunnelInterfaceStateListener;
 144     private VpnInterfaceOpListener vpnInterfaceOpListener;
 145     private ArpNotificationHandler arpNotificationHandler;
 146     private DpnInVpnChangeListener dpnInVpnChangeListener;
 147     private NotificationPublishService notificationPublishService;
 148     private FibRpcService fibService;
 149
 150     /**
 151      * Responsible for listening to data change related to VPN Interface
 152      * Bind VPN Service on the interface and informs the BGP service
 153      *
 154      * @param db - dataBroker service reference
 155      * @param bgpManager Used to advertise routes to the BGP Router
 156      * @param notificationService Used to subscribe to notification events
 157      */
 158     public VpnInterfaceManager(final DataBroker db, final IBgpManager bgpManager, NotificationService notificationService) {
 159         super(VpnInterface.class);
 160         broker = db;
 161         this.bgpManager = bgpManager;
 162         interfaceListener = new InterfaceStateChangeListener(db, this);
 163         subnetRouteInterfaceListener = new SubnetRouteInterfaceStateChangeListener(db, this);
 164         vpnInterfaceOpListener = new VpnInterfaceOpListener();
 165         arpNotificationHandler = new ArpNotificationHandler(this, broker);
 166         vpnSubnetRouteHandler = new VpnSubnetRouteHandler(broker, bgpManager, this);
 167         dpnInVpnChangeListener = new DpnInVpnChangeListener(broker);
 168         notificationService.registerNotificationListener(vpnSubnetRouteHandler);
 169         notificationService.registerNotificationListener(arpNotificationHandler);
 170         notificationService.registerNotificationListener(dpnInVpnChangeListener);
 171         registerListener(db);
 172     }
 173
 174     public void setMdsalManager(IMdsalApiManager mdsalManager) {
 175         this.mdsalManager = mdsalManager;
 176     }
 177
 178     public void setIfaceMgrRpcService(OdlInterfaceRpcService ifMgrRpcService) {
 179         this.ifaceMgrRpcService = ifMgrRpcService;
 180         interfaceListener.setIfaceMgrRpcService(ifMgrRpcService);
 181     }
 182
 183     public void setFibManager(IFibManager fibManager) {
 184         this.fibManager = fibManager;
 185     }
 186
 187     public IFibManager getFibManager() {
 188         return this.fibManager;
 189     }
 190
 191
 192     public void setIdManager(IdManagerService idManager) {
 193         this.idManager = idManager;
 194         vpnSubnetRouteHandler.setIdManager(idManager);
 195     }
 196
 197     public void setArpManager(OdlArputilService arpManager) {
 198         this.arpManager = arpManager;
 199     }
 200
 201     void setNotificationPublishService(NotificationPublishService notificationPublishService) {
 202         this.notificationPublishService = notificationPublishService;
 203     }
 204
 205     public void setNeutronvpnManager(NeutronvpnService neuService) { this.neuService = neuService; }
 206
 207     public void setFibRpcService(FibRpcService fibService) {
 208         this.fibService = fibService;
 209     }
 210
 211     public FibRpcService getFibRpcService() {
 212         return fibService;
 213     }
 214
 215     public VpnSubnetRouteHandler getVpnSubnetRouteHandler() {
 216         return this.vpnSubnetRouteHandler;
 217     }
 218
 219     @Override
 220     public void close() throws Exception {
 221         if (listenerRegistration != null) {
 222             try {
 223                 listenerRegistration.close();
 224                 opListenerRegistration.close();
 225             } catch (final Exception e) {
 226                 LOG.error("Error when cleaning up DataChangeListener.", e);
 227             }
 228             listenerRegistration = null;
 229             opListenerRegistration = null;
 230         }
 231         LOG.info("VPN Interface Manager Closed");
 232     }
 233
 234     private void registerListener(final DataBroker db) {
 235         try {
 236             listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 237                     getWildCardPath(), VpnInterfaceManager.this, DataChangeScope.SUBTREE);
 238             opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
 239                     getWildCardPath(), vpnInterfaceOpListener, DataChangeScope.SUBTREE);
 240         } catch (final Exception e) {
 241             LOG.error("VPN Service DataChange listener registration fail!", e);
 242             throw new IllegalStateException("VPN Service registration Listener failed.", e);
 243         }
 244     }
 245
 246     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 247         return InstanceIdentifier.create(InterfacesState.class)
 248                 .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 249     }
 250
 251     @Override
 252     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 253         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 254         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 255         final String interfaceName = key.getName();
 256
 257         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 258                 InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 259         if(interfaceState != null){
 260             try{
 261                 final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 262                 final int ifIndex = interfaceState.getIfIndex();
 263                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 264                 dataStoreCoordinator.enqueueJob("VPNINTERFACE-"+ interfaceName,
 265                         new Callable<List<ListenableFuture<Void>>>() {
 266                             @Override
 267                             public List<ListenableFuture<Void>> call() throws Exception {
 268                                 WriteTransaction writeConfigTxn = broker.newWriteOnlyTransaction();
 269                                 WriteTransaction writeOperTxn = broker.newWriteOnlyTransaction();
 270                                                             WriteTransaction writeInvTxn = broker.newWriteOnlyTransaction();
 271                                 processVpnInterfaceUp(dpnId, vpnInterface, ifIndex, false, writeConfigTxn, writeOperTxn, writeInvTxn);
 272                                 CheckedFuture<Void, TransactionCommitFailedException> futures = writeOperTxn.submit();
 273                                 try {
 274                                     futures.get();
 275                                 } catch (InterruptedException | ExecutionException e) {
 276                                     LOG.error("Error adding oper data for interface {} to vpn {} on dpn {}", interfaceName,
 277                                             vpnInterface.getVpnInstanceName(), dpnId);
 278                                     throw new RuntimeException(e.getMessage());
 279                                 }
 280                                 futures = writeConfigTxn.submit();
 281                                 try {
 282                                     futures.get();
 283                                 } catch (InterruptedException | ExecutionException e) {
 284                                                                         LOG.error("Error adding config data for interface {} to vpn {} on dpn {}", interfaceName,
 285                                         vpnInterface.getVpnInstanceName(), dpnId);
 286                                                                         throw new RuntimeException(e.getMessage());
 287                                                                 }
 288                                                                 futures = writeInvTxn.submit();
 289                                                                 try {
 290                                                                         futures.get();
 291                                                                 } catch (InterruptedException | ExecutionException e) {
 292                                                                         LOG.error("Error adding inventory/flow data for interface {} to vpn {} on dpn {}", interfaceName,
 293                                         vpnInterface.getVpnInstanceName(), dpnId);
 294                                                                         throw new RuntimeException(e.getMessage());
 295                                                                 }
 296                                                                 return null;
 297                             }
 298                         });
 299             }catch (Exception e){
 300                 LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", interfaceName, e);
 301                 return;
 302             }
 303         } else {
 304             LOG.info("Handling addition of VPN interface {} skipped as interfaceState is not available", interfaceName);
 305         }
 306     }
 307
 308     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface,
 309                                          final int lPortTag, boolean isInterfaceUp,
 310                                          WriteTransaction writeConfigTxn,
 311                                          WriteTransaction writeOperTxn,
 312                                                                                  WriteTransaction writeInvTxn) {
 313
 314         final String interfaceName = vpnInterface.getName();
 315         if (!isInterfaceUp) {
 316             final String vpnName = vpnInterface.getVpnInstanceName();
 317             LOG.info("Binding vpn service to interface {} ", interfaceName);
 318             long vpnId = VpnUtil.getVpnId(broker, vpnName);
 319             if (vpnId == VpnConstants.INVALID_ID) {
 320                 LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 321                 return;
 322             }
 323             boolean waitForVpnInterfaceOpRemoval = false;
 324             VpnInterface opVpnInterface = VpnUtil.getOperationalVpnInterface(broker, vpnInterface.getName());
 325             if (opVpnInterface != null ) {
 326                 String opVpnName = opVpnInterface.getVpnInstanceName();
 327                 String primaryInterfaceIp = null;
 328                 if(opVpnName.equals(vpnName)) {
 329                     // Please check if the primary VRF Entry does not exist for VPNInterface
 330                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 331                     // back to back
 332                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 333                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(broker, interfaceName);
 334                     if (adjs == null) {
 335                         LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained", interfaceName);
 336                         return;
 337                     }
 338                     for (Adjacency adj : adjs) {
 339                         if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
 340                             primaryInterfaceIp = adj.getIpAddress();
 341                             break;
 342                         }
 343                     }
 344                     if (primaryInterfaceIp == null) {
 345                         LOG.info("VPN Interface {} addition failed as primary adjacency "
 346                                 + "for this vpn interface could not be obtained", interfaceName);
 347                         return;
 348                     }
 349                     // Get the rd of the vpn instance
 350                     String rd = getRouteDistinguisher(opVpnName);
 351                                         rd =  (rd == null) ? opVpnName : rd;
 352                     VrfEntry vrf = VpnUtil.getVrfEntry(broker, rd, primaryInterfaceIp);
 353                     if (vrf != null) {
 354                         LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
 355                         return;
 356                     }
 357                     waitForVpnInterfaceOpRemoval = true;
 358                 } else {
 359                     LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
 360                             interfaceName, vpnName, opVpnName);
 361                 }
 362             }
 363             if (!waitForVpnInterfaceOpRemoval) {
 364                 // Add the VPNInterface and quit
 365                 updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 366                 bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 367                 processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 368                 return;
 369             }
 370
 371             // FIB didn't get a chance yet to clean up this VPNInterface
 372             // Let us give it a chance here !
 373             LOG.info("VPN Interface {} waiting for FIB to clean up! ", interfaceName);
 374             try {
 375                 Runnable notifyTask = new VpnNotifyTask();
 376                 vpnIntfMap.put(interfaceName, notifyTask);
 377                 synchronized (notifyTask) {
 378                     try {
 379                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 380                     } catch (InterruptedException e) {
 381                     }
 382                 }
 383             } finally {
 384                 vpnIntfMap.remove(interfaceName);
 385             }
 386
 387             opVpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
 388             if (opVpnInterface != null) {
 389                 LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
 390                 return;
 391             }
 392             // VPNInterface got removed, proceed with Add
 393             updateVpnToDpnMapping(dpId, vpnName, interfaceName, true /* add */);
 394             bindService(dpId, vpnName, interfaceName, lPortTag, writeConfigTxn, writeInvTxn);
 395             processVpnInterfaceAdjacencies(dpId, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
 396         } else {
 397             // Interface is retained in the DPN, but its Link Up.
 398             // Advertise prefixes again for this interface to BGP
 399             advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
 400                     vpnInterface);
 401         }
 402     }
 403
 404
 405 //    private class UpdateDpnToVpnWorker implements Callable<List<ListenableFuture<Void>>> {
 406 //        BigInteger dpnId;
 407 //        String vpnName;
 408 //        String interfaceName;
 409 //        boolean addToDpn;
 410 //        int lPortTag;
 411 //
 412 //        public UpdateDpnToVpnWorker(BigInteger dpnId, String vpnName, String interfaceName,
 413 //                                    int lPortTag, boolean addToDpn) {
 414 //            this.dpnId= dpnId;
 415 //            this.vpnName = vpnName;
 416 //            this.interfaceName = interfaceName;
 417 //            this.lPortTag = lPortTag;
 418 //            this.addToDpn = addToDpn;
 419 //        }
 420 //
 421 //        @Override
 422 //        public List<ListenableFuture<Void>> call() throws Exception {
 423 //            // If another renderer(for eg : CSS) needs to be supported, check can be performed here
 424 //            // to call the respective helpers.
 425 //            WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
 426 //            updateDpnDbs(dpnId, vpnName, interfaceName, addToDpn, writeTxn);
 427 //            List<ListenableFuture<Void>> futures = new ArrayList<>();
 428 //            futures.add(writeTxn.submit());
 429 //            ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
 430 //            Futures.addCallback(listenableFuture,
 431 //                    new UpdateDpnToVpnCallback(dpnId, vpnName, interfaceName, lPortTag, addToDpn));
 432 //            return futures;
 433 //        }
 434 //    }
 435 //
 436 //
 437 //    /**
 438 //     * JobCallback class is used as a future callback for
 439 //     * main and rollback workers to handle success and failure.
 440 //     */
 441 //    private class UpdateDpnToVpnCallback implements FutureCallback<List<Void>> {
 442 //        BigInteger dpnId;
 443 //        String vpnName;
 444 //        String interfaceName;
 445 //        boolean addToDpn;
 446 //        int lPortTag;
 447 //
 448 //        public UpdateDpnToVpnCallback(BigInteger dpnId, String vpnName, String interfaceName,
 449 //                                      int lPortTag, boolean addToDpn) {
 450 //            this.dpnId= dpnId;
 451 //            this.vpnName = vpnName;
 452 //            this.interfaceName = interfaceName;
 453 //            this.lPortTag = lPortTag;
 454 //            this.addToDpn = addToDpn;
 455 //        }
 456 //
 457 //        /**
 458 //         * @param voids
 459 //         * This implies that all the future instances have returned success. -- TODO: Confirm this
 460 //         */
 461 //        @Override
 462 //        public void onSuccess(List<Void> voids) {
 463 //            WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
 464 //            bindService(dpnId, vpnName, interfaceName, lPortTag, writeTxn);
 465 //            processVpnInterfaceAdjacencies(dpnId, vpnName, interfaceName, writeTxn);
 466 //            writeTxn.submit();
 467 //        }
 468 //
 469 //        /**
 470 //         *
 471 //         * @param throwable
 472 //         * This method is used to handle failure callbacks.
 473 //         * If more retry needed, the retrycount is decremented and mainworker is executed again.
 474 //         * After retries completed, rollbackworker is executed.
 475 //         * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 476 //         */
 477 //
 478 //        @Override
 479 //        public void onFailure(Throwable throwable) {
 480 //            LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 481 //        }
 482 //    }
 483
 484
 485     private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
 486                                                  VpnInterface intf) {
 487         //Read NextHops
 488         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 489         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 490
 491         String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
 492         if (rd == null) {
 493             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 494                     intf.getName(), intf.getVpnInstanceName());
 495             return;
 496         } else {
 497             if (rd.equals(intf.getVpnInstanceName())) {
 498                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in " +
 499                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 500
 501                 return;
 502             }
 503         }
 504         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
 505                 intf.getVpnInstanceName(), rd);
 506
 507         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 508         if (nextHopIp == null){
 509             LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
 510             return;
 511         }
 512
 513         if (adjacencies.isPresent()) {
 514             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 515
 516             if (!nextHops.isEmpty()) {
 517                 LOG.trace("NextHops are " + nextHops);
 518                 for (Adjacency nextHop : nextHops) {
 519                     long label = nextHop.getLabel();
 520                     try {
 521                         LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 522                         bgpManager.advertisePrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
 523                         LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 524                     } catch(Exception e) {
 525                         LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
 526                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 527                     }
 528                 }
 529             }
 530         }
 531     }
 532
 533     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 534         //Read NextHops
 535         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 536         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 537
 538         String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
 539         if (rd == null) {
 540             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 541                     intf.getName(), intf.getVpnInstanceName());
 542             return;
 543         } else {
 544             if (rd.equals(intf.getVpnInstanceName())) {
 545                 LOG.info("withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in " +
 546                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 547                 return;
 548             }
 549         }
 550         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 551                 intf.getVpnInstanceName(), rd);
 552         if (adjacencies.isPresent()) {
 553             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 554
 555             if (!nextHops.isEmpty()) {
 556                 LOG.trace("NextHops are " + nextHops);
 557                 for (Adjacency nextHop : nextHops) {
 558                     try {
 559                         LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 560                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 561                         LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 562                     } catch(Exception e) {
 563                         LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
 564                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 565                     }
 566                 }
 567             }
 568         }
 569     }
 570
 571     private void updateVpnToDpnMapping(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 572         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 573         if (dpId == null) {
 574             dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, interfaceName);
 575         }
 576         if(!dpId.equals(BigInteger.ZERO)) {
 577             if(add)
 578                 createOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 579             else
 580                 removeOrUpdateVpnToDpnList(vpnId, dpId, interfaceName, vpnName);
 581         }
 582     }
 583
 584     private void bindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName, int lPortTag,
 585                              WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
 586         int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 587         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 588
 589         int instructionKey = 0;
 590         List<Instruction> instructions = new ArrayList<Instruction>();
 591
 592         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(MetaDataUtil.getVpnIdMetadata(vpnId),
 593                 MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 594         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 595
 596         BoundServices
 597                 serviceInfo =
 598                 InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 599                         NwConstants.L3VPN_SERVICE_INDEX, priority,
 600                         NwConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 601         writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION,
 602                 InterfaceUtils.buildServiceId(vpnInterfaceName, NwConstants.L3VPN_SERVICE_INDEX), serviceInfo, true);
 603         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
 604                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW, writeInvTxn);
 605         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
 606                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.ADD_FLOW, writeInvTxn);
 607
 608     }
 609
 610     private void processVpnInterfaceAdjacencies(BigInteger dpnId, String vpnName, String interfaceName,
 611                                                 WriteTransaction writeConfigTxn,
 612                                                 WriteTransaction writeOperTxn) {
 613         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 614         // Read NextHops
 615         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 616         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
 617
 618         if (adjacencies.isPresent()) {
 619             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 620             List<Adjacency> value = new ArrayList<>();
 621
 622             // Get the rd of the vpn instance
 623             String rd = getRouteDistinguisher(vpnName);
 624
 625             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 626             if (nextHopIp == null){
 627                 LOG.error("NextHop for interface {} is null", interfaceName);
 628                 return;
 629             }
 630
 631             List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
 632
 633             LOG.trace("NextHops for interface {} are {}", interfaceName, nextHops);
 634             for (Adjacency nextHop : nextHops) {
 635                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 636                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 637                         VpnUtil.getNextHopLabelKey((rd == null) ? vpnName
 638                                 : rd, prefix));
 639                 List<String> adjNextHop = nextHop.getNextHopIpList();
 640                 value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIpList(
 641                         (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : Arrays.asList(nextHopIp))
 642                         .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 643
 644                 if (nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 645                     LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, interfaceName, vpnName);
 646                     writeOperTxn.merge(
 647                             LogicalDatastoreType.OPERATIONAL,
 648                             VpnUtil.getPrefixToInterfaceIdentifier(
 649                                     VpnUtil.getVpnId(broker, vpnName), prefix),
 650                             VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix), true);
 651                 } else {
 652                     //Extra route adjacency
 653                     LOG.trace("Adding prefix {} and nexthopList {} as extra-route for vpn", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName);
 654                     writeOperTxn.merge(
 655                             LogicalDatastoreType.OPERATIONAL,
 656                             VpnUtil.getVpnToExtrarouteIdentifier(
 657                                     (rd != null) ? rd : vpnName, nextHop.getIpAddress()),
 658                             VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()), true);
 659                 }
 660             }
 661
 662             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 663
 664             VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 665             InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 666             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, true);
 667             long vpnId = VpnUtil.getVpnId(broker, vpnName);
 668
 669             for (Adjacency nextHop : aug.getAdjacency()) {
 670                 long label = nextHop.getLabel();
 671                 List<String> nextHopList = new ArrayList<>(nextHop.getNextHopIpList());
 672                 if (rd != null) {
 673                     addToLabelMapper(label, dpnId, nextHop.getIpAddress(), nextHopList, vpnId,
 674                             interfaceName, null,false, rd, writeOperTxn);
 675                     addPrefixToBGP(rd, nextHop.getIpAddress(), nextHopIp, label, writeConfigTxn);
 676                     //TODO: ERT - check for VPNs importing my route
 677                     for (VpnInstance vpn : vpnsToImportRoute) {
 678                         String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
 679                         if (vpnRd != null) {
 680                             LOG.debug("Exporting route with rd {} prefix {} nexthop {} label {} to VPN {}", vpnRd, nextHop.getIpAddress(), nextHopIp, label, vpn);
 681                             fibManager.addOrUpdateFibEntry(broker, vpnRd, nextHop.getIpAddress(), Arrays.asList(nextHopIp), (int) label,
 682                                     RouteOrigin.SELF_IMPORTED, writeConfigTxn);
 683                         }
 684                     }
 685                 } else {
 686                     // ### add FIB route directly
 687                     fibManager.addOrUpdateFibEntry(broker, vpnName, nextHop.getIpAddress(), Arrays.asList(nextHopIp),
 688                             (int) label, RouteOrigin.STATIC, writeConfigTxn);
 689                 }
 690             }
 691         }
 692     }
 693
 694     private List<VpnInstance> getVpnsImportingMyRoute(final String vpnName) {
 695         List<VpnInstance> vpnsToImportRoute = new ArrayList<>();
 696
 697         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 698                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 699         Optional<VpnInstance> optVpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 700         final VpnInstance vpnInstance;
 701         if (optVpnInstance.isPresent()) {
 702             vpnInstance = optVpnInstance.get();
 703         } else {
 704             LOG.debug("Could not retrieve vpn instance {} to check for vpns importing the routes", vpnName);
 705             return vpnsToImportRoute;
 706         }
 707
 708         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 709             @Override
 710             public boolean apply(VpnInstance input) {
 711                 return !input.getVpnInstanceName().equals(vpnName);
 712             }
 713         };
 714
 715         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 716             @Override
 717             public boolean apply(VpnInstance input) {
 718                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ExportExtcommunity),
 719                         getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 720                 return Iterators.size(commonRTs.iterator()) > 0;
 721             }
 722         };
 723
 724         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 725             @Override
 726             public String apply(VpnInstance vpnInstance) {
 727                 //return vpnInstance.getVpnInstanceName();
 728                 return vpnInstance.getIpv4Family().getRouteDistinguisher();
 729             }
 730         };
 731
 732         vpnsToImportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(broker)).
 733                 filter(excludeVpn).
 734                 filter(matchRTs).toList();
 735         return vpnsToImportRoute;
 736     }
 737
 738     private List<VpnInstance> getVpnsExportingMyRoute(final String vpnName) {
 739         List<VpnInstance> vpnsToExportRoute = new ArrayList<>();
 740
 741         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 742                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 743         Optional<VpnInstance> optVpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 744         final VpnInstance vpnInstance;
 745         if (optVpnInstance.isPresent()) {
 746             vpnInstance = optVpnInstance.get();
 747         } else {
 748             LOG.debug("Could not retrieve vpn instance {} to check for vpns exporting the routes", vpnName);
 749             return vpnsToExportRoute;
 750         }
 751
 752         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 753             @Override
 754             public boolean apply(VpnInstance input) {
 755                 return !input.getVpnInstanceName().equals(vpnName);
 756             }
 757         };
 758
 759         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 760             @Override
 761             public boolean apply(VpnInstance input) {
 762                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ImportExtcommunity),
 763                         getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
 764                 return Iterators.size(commonRTs.iterator()) > 0;
 765             }
 766         };
 767
 768         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 769             @Override
 770             public String apply(VpnInstance vpnInstance) {
 771                 return vpnInstance.getVpnInstanceName();
 772             }
 773         };
 774
 775         vpnsToExportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(broker)).
 776                 filter(excludeVpn).
 777                 filter(matchRTs).toList();
 778         return vpnsToExportRoute;
 779     }
 780
 781     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
 782         final Predicate<T> inPredicate = Predicates.<T>in(collection2);
 783         return new Iterable<T>() {
 784             @Override
 785             public Iterator<T> iterator() {
 786                 return Iterators.filter(collection1.iterator(), inPredicate);
 787             }
 788         };
 789     }
 790
 791     private List<String> getRts(VpnInstance vpnInstance, VpnTarget.VrfRTType rtType) {
 792         String name = vpnInstance.getVpnInstanceName();
 793         List<String> rts = new ArrayList<>();
 794         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 795         if (vpnConfig == null) {
 796             LOG.trace("vpn config is not available for {}", name);
 797             return rts;
 798         }
 799         VpnTargets targets = vpnConfig.getVpnTargets();
 800         if (targets == null) {
 801             LOG.trace("vpn targets not available for {}", name);
 802             return rts;
 803         }
 804         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 805         if (vpnTargets == null) {
 806             LOG.trace("vpnTarget values not available for {}", name);
 807             return rts;
 808         }
 809         for (VpnTarget target : vpnTargets) {
 810             //TODO: Check for RT type is Both
 811             if(target.getVrfRTType().equals(rtType) ||
 812                     target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
 813                 String rtValue = target.getVrfRTValue();
 814                 rts.add(rtValue);
 815             }
 816         }
 817         return rts;
 818     }
 819
 820     private List<String> getExportRts(VpnInstance vpnInstance) {
 821         List<String> exportRts = new ArrayList<>();
 822         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 823         VpnTargets targets = vpnConfig.getVpnTargets();
 824         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 825         for (VpnTarget target : vpnTargets) {
 826             if (target.getVrfRTType().equals(VpnTarget.VrfRTType.ExportExtcommunity)) {
 827                 String rtValue = target.getVrfRTValue();
 828                 exportRts.add(rtValue);
 829             }
 830         }
 831         return exportRts;
 832     }
 833
 834     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 835                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow,
 836                              WriteTransaction writeConfigTxn){
 837         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 838         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, MetaDataUtil.getVpnIdMetadata(vpnId));
 839         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 840                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 841
 842         // Matching Arp reply flows
 843         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 844         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 845                 metadata, metadataMask }));
 846
 847         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 848
 849         // Instruction to punt to controller
 850         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 851         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 852         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 853         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
 854                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
 855
 856         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 857
 858         // Install the flow entry in L3_INTERFACE_TABLE
 859         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 860                 NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 861         FlowEntity flowEntity;
 862         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 863                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 864                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 865
 866         Flow flow = flowEntity.getFlowBuilder().build();
 867         String flowId = flowEntity.getFlowId();
 868         FlowKey flowKey = new FlowKey( new FlowId(flowId));
 869         Node nodeDpn = buildDpnNode(dpId);
 870
 871         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 872                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 873                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 874
 875         if (writeConfigTxn != null) {
 876             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 877                 LOG.debug("Creating ARP Flow for interface {}", vpnInterfaceName);
 878                 writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
 879             } else {
 880                 LOG.debug("Deleting ARP Flow for interface {}", vpnInterfaceName);
 881                 writeConfigTxn.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 882             }
 883         } else {
 884             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 885                 LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 886                 mdsalManager.installFlow(flowEntity);
 887             } else {
 888                 LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 889                 mdsalManager.removeFlow(flowEntity);
 890             }
 891         }
 892     }
 893
 894     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
 895     private Node buildDpnNode(BigInteger dpnId) {
 896         NodeId nodeId = new NodeId("openflow:" + dpnId);
 897         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
 898
 899         return nodeDpn;
 900     }
 901
 902     private String getRouteDistinguisher(String vpnName) {
 903         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 904                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 905         Optional<VpnInstance> vpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 906         String rd = "";
 907         if(vpnInstance.isPresent()) {
 908             VpnInstance instance = vpnInstance.get();
 909             VpnAfConfig config = instance.getIpv4Family();
 910             rd = config.getRouteDistinguisher();
 911         }
 912         return rd;
 913     }
 914
 915     private void createOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 916         String routeDistinguisher = getRouteDistinguisher(vpnName);
 917         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 918         Boolean newDpnOnVpn = Boolean.FALSE;
 919
 920         synchronized (vpnName.intern()) {
 921             WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
 922             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 923             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 924             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data
 925                     .entry.vpn.to.dpn.list.VpnInterfaces
 926                     vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 927
 928             if (dpnInVpn.isPresent()) {
 929                 if (writeTxn != null) {
 930                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id.child(
 931                             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance
 932                                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 933                             new VpnInterfacesKey(intfName)), vpnInterface, true);
 934                 } else {
 935                     VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 936                             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance
 937                                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 938                             new VpnInterfacesKey(intfName)), vpnInterface);
 939                 }
 940             } else {
 941                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 942                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = new ArrayList<>();
 943                 vpnInterfaces.add(vpnInterface);
 944                 VpnToDpnListBuilder vpnToDpnListBuilder = new VpnToDpnListBuilder().setDpnId(dpnId);
 945                 vpnToDpnListBuilder.setDpnState(VpnToDpnList.DpnState.Active).setVpnInterfaces(vpnInterfaces);
 946
 947                 if (writeTxn != null) {
 948                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build(), true);
 949                 } else {
 950                     VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnListBuilder.build());
 951                 }
 952                 newDpnOnVpn = Boolean.TRUE;
 953             }
 954             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
 955             try {
 956                 futures.get();
 957             } catch (InterruptedException | ExecutionException e) {
 958                 LOG.error("Error adding to dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
 959                 throw new RuntimeException(e.getMessage());
 960             }
 961              /*
 962             * Informing the Fib only after writeTxn is submitted successfuly.
 963             */
 964             if (newDpnOnVpn) {
 965                 LOG.debug("Sending populateFib event for new dpn {} in VPN {}", dpnId, vpnName);
 966                 fibManager.populateFibOnNewDpn(dpnId, vpnId, rd);
 967                 publishAddNotification(dpnId, vpnName, rd);
 968             }
 969         }
 970     }
 971
 972     private void removeOrUpdateVpnToDpnList(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 973         Boolean lastDpnOnVpn = Boolean.FALSE;
 974
 975         synchronized (vpnName.intern()) {
 976             String rd = VpnUtil.getVpnRd(broker, vpnName);
 977             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 978             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 979             WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
 980             if (dpnInVpn.isPresent()) {
 981                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 982                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 983                 org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 984                         currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 985
 986                 if (vpnInterfaces.remove(currVpnInterface)) {
 987                     if (vpnInterfaces.isEmpty()) {
 988                         List<IpAddresses> ipAddresses = dpnInVpn.get().getIpAddresses();
 989                         if (ipAddresses == null || ipAddresses.isEmpty()) {
 990                             VpnToDpnListBuilder dpnInVpnBuilder =
 991                                     new VpnToDpnListBuilder(dpnInVpn.get())
 992                                             .setDpnState(VpnToDpnList.DpnState.Inactive)
 993                                             .setVpnInterfaces(null);
 994                             if (writeTxn != null) {
 995                                 writeTxn.put(LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build(), true);
 996                             } else {
 997                                 VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id, dpnInVpnBuilder.build());
 998                             }
 999                             lastDpnOnVpn = Boolean.TRUE;
1000                         } else {
1001                             LOG.warn("vpn interfaces are empty but ip addresses are present for the vpn {} in dpn {}", vpnName, dpnId);
1002                         }
1003                     } else {
1004                         if (writeTxn != null) {
1005                             writeTxn.delete(LogicalDatastoreType.OPERATIONAL, id.child(
1006                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1007                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1008                                     new VpnInterfacesKey(intfName)));
1009                         } else {
1010                             VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
1011                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1012                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1013                                     new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
1014                         }
1015                     }
1016                 }
1017             }
1018             CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
1019             try {
1020                 futures.get();
1021             } catch (InterruptedException | ExecutionException e) {
1022                 LOG.error("Error removing from dpnToVpnList for vpn {} interface {} dpn {}", vpnName, intfName, dpnId);
1023                 throw new RuntimeException(e.getMessage());
1024             }
1025             if (lastDpnOnVpn) {
1026                 LOG.debug("Sending cleanup event for dpn {} in VPN {}", dpnId, vpnName);
1027                 fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
1028                 publishRemoveNotification(dpnId, vpnName, rd);
1029             }
1030         }
1031     }
1032
1033     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1034         List<VpnInstance> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1035         for (VpnInstance vpn : vpnsToExportRoute) {
1036             String rd = vpn.getIpv4Family().getRouteDistinguisher();
1037             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(broker, vpn.getIpv4Family().getRouteDistinguisher());
1038             WriteTransaction writeConfigTxn = broker.newWriteOnlyTransaction();
1039             if (vrfEntries != null) {
1040                 for (VrfEntry vrfEntry : vrfEntries) {
1041                     try {
1042                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1043                             continue;
1044                         }
1045                         String prefix = vrfEntry.getDestPrefix();
1046                         long label = vrfEntry.getLabel();
1047                         List<String> nextHops = vrfEntry.getNextHopAddressList();
1048                         SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1049                         for (String nh : nextHops) {
1050                             if (route != null) {
1051                                 LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1052                                 importSubnetRouteForNewVpn(rd, prefix, nh, (int)label, route, writeConfigTxn);
1053                             } else {
1054                                 LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1055                                 fibManager.addOrUpdateFibEntry(broker, vpnRd, prefix, Arrays.asList(nh), (int)label,
1056                                         RouteOrigin.SELF_IMPORTED, writeConfigTxn);
1057                             }
1058                         }
1059                     } catch (Exception e) {
1060                         LOG.error("Exception occurred while importing route with prefix {} label {} nexthop {} from vpn {} to vpn {}", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpn.getVpnInstanceName(), vpnName);
1061                     }
1062                 }
1063                 writeConfigTxn.submit();
1064             } else {
1065                 LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getIpv4Family().getRouteDistinguisher());
1066             }
1067         }
1068     }
1069
1070     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label, WriteTransaction writeConfigTxn) {
1071         try {
1072             LOG.info("ADD: Adding Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1073             fibManager.addOrUpdateFibEntry(broker, rd, prefix, Arrays.asList(nextHopIp), (int)label, RouteOrigin.STATIC, writeConfigTxn);
1074             bgpManager.advertisePrefix(rd, prefix, Arrays.asList(nextHopIp), (int)label);
1075             LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1076         } catch(Exception e) {
1077             LOG.error("Add prefix failed", e);
1078         }
1079     }
1080
1081
1082     private InstanceIdentifier<VpnInterface> getWildCardPath() {
1083         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
1084     }
1085
1086     @Override
1087     public void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1088         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
1089         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1090         final String interfaceName = key.getName();
1091
1092         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1093         final Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1094         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
1095                 InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
1096         if (optVpnInterface.isPresent()){
1097             BigInteger dpnId = BigInteger.ZERO;
1098             Boolean dpnIdRetrieved = Boolean.FALSE;
1099             if(interfaceState != null){
1100                 try{
1101                     dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1102                     dpnIdRetrieved = Boolean.TRUE;
1103                 }catch (Exception e){
1104                     LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", interfaceName, e);
1105                 }
1106             } else {
1107                 LOG.error("Unable to retrieve interfaceState for interface {} , quitting ", interfaceName);
1108                 return;
1109             }
1110             final VpnInterface vpnOpInterface = optVpnInterface.get();
1111             if(dpnIdRetrieved == Boolean.FALSE){
1112                 LOG.info("dpnId for {} has not been retrieved yet. Fetching from vpn interface operational DS", interfaceName);
1113                 dpnId = vpnOpInterface.getDpnId();
1114             }
1115             final int ifIndex = interfaceState.getIfIndex();
1116             final BigInteger dpId = dpnId;
1117             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1118             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1119                     new Callable<List<ListenableFuture<Void>>>() {
1120                         @Override
1121                         public List<ListenableFuture<Void>> call() throws Exception {
1122                             WriteTransaction writeConfigTxn = broker.newWriteOnlyTransaction();
1123                             WriteTransaction writeOperTxn = broker.newWriteOnlyTransaction();
1124                             WriteTransaction writeInvTxn = broker.newWriteOnlyTransaction();
1125                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, false, true, writeConfigTxn, writeOperTxn, writeInvTxn);
1126                             CheckedFuture<Void, TransactionCommitFailedException> futures = writeOperTxn.submit();
1127                             try {
1128                                 futures.get();
1129                             } catch (InterruptedException | ExecutionException e) {
1130                                 LOG.error("Error removing Oper data for interface {} from vpn {} on dpn {}", interfaceName,
1131                                         vpnOpInterface.getVpnInstanceName(), dpId);
1132                                 throw new RuntimeException(e.getMessage());
1133                             }
1134                             futures = writeConfigTxn.submit();
1135                             try {
1136                                 futures.get();
1137                             } catch (InterruptedException | ExecutionException e) {
1138                                 LOG.error("Error removing Config data for interface {} from vpn {} on dpn {}", interfaceName,
1139                                         vpnOpInterface.getVpnInstanceName(), dpId);
1140                                 throw new RuntimeException(e.getMessage());
1141                             }
1142                             futures = writeInvTxn.submit();
1143                             try {
1144                                 futures.get();
1145                             } catch (InterruptedException | ExecutionException e) {
1146                                 LOG.error("Error removing Inventory/Flow data for interface {} from vpn {} on dpn {}", interfaceName,
1147                                         vpnOpInterface.getVpnInstanceName(), dpId);
1148                                 throw new RuntimeException(e.getMessage());
1149                                                         }
1150                             return null;
1151                         }
1152                     });
1153
1154         }else{
1155             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
1156         }
1157     }
1158
1159     protected void processVpnInterfaceDown(BigInteger dpId,
1160                                                                                    String interfaceName,
1161                                                                                    int lPortTag,
1162                                                                                    boolean isInterfaceStateDown,
1163                                            boolean isConfigRemoval,
1164                                                                                    WriteTransaction writeConfigTxn,
1165                                                                                    WriteTransaction writeOperTxn,
1166                                                                                    WriteTransaction writeInvTxn) {
1167         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1168         if (!isInterfaceStateDown) {
1169             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
1170             if(vpnInterface == null){
1171                 LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
1172                 return;
1173             }else{
1174                 final String vpnName = vpnInterface.getVpnInstanceName();
1175                 if(!vpnInterface.isScheduledForRemove()){
1176                     VpnUtil.scheduleVpnInterfaceForRemoval(broker, interfaceName, dpId, vpnName, Boolean.TRUE, writeOperTxn);
1177                     removeAdjacenciesFromVpn(dpId, interfaceName, vpnInterface.getVpnInstanceName(), writeConfigTxn);
1178                     LOG.info("Unbinding vpn service from interface {} ", interfaceName);
1179                     unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown, isConfigRemoval, writeConfigTxn, writeInvTxn);
1180                 }else{
1181                     LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ", interfaceName);
1182                     return;
1183                 }
1184             }
1185         } else {
1186             // Interface is retained in the DPN, but its Link Down.
1187             // Only withdraw the prefixes for this interface from BGP
1188             VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
1189             if(vpnInterface == null){
1190                 LOG.info("Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in operational data store", interfaceName);
1191                 return;
1192             }else {
1193                 withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
1194             }
1195         }
1196     }
1197
1198     private void waitForFibToRemoveVpnPrefix(String interfaceName) {
1199         // FIB didn't get a chance yet to clean up this VPNInterface
1200         // Let us give it a chance here !
1201         LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
1202         try {
1203             Runnable notifyTask = new VpnNotifyTask();
1204             vpnIntfMap.put(interfaceName, notifyTask);
1205             synchronized (notifyTask) {
1206                 try {
1207                     notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
1208                 } catch (InterruptedException e) {
1209                 }
1210             }
1211         } finally {
1212             vpnIntfMap.remove(interfaceName);
1213         }
1214     }
1215
1216     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final String interfaceName, final String vpnName,
1217                                           WriteTransaction writeConfigTxn) {
1218         //Read NextHops
1219         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1220         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1221         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
1222
1223         String rd = VpnUtil.getVpnRd(broker, vpnName);
1224         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", interfaceName,
1225                 vpnName, rd);
1226         if (adjacencies.isPresent()) {
1227             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1228
1229             if (!nextHops.isEmpty()) {
1230                 LOG.trace("NextHops are " + nextHops);
1231                 for (Adjacency nextHop : nextHops) {
1232                     List<String> nhList = new ArrayList<String>();
1233                     if (nextHop.getMacAddress() == null || nextHop.getMacAddress().isEmpty()) {
1234                         // This is either an extra-route (or) a learned IP via subnet-route
1235                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
1236                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1237                             LOG.error("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
1238                                     rd, nextHop.getIpAddress());
1239                             continue;
1240                         }
1241                         nhList = Arrays.asList(nextHopIp);
1242                     } else {
1243                         // This is a primary adjacency
1244                         nhList = nextHop.getNextHopIpList();
1245                     }
1246                     if (rd.equals(vpnName)) {
1247                         //this is an internal vpn - the rd is assigned to the vpn instance name;
1248                         //remove from FIB directly
1249                         for(String nh : nhList) {
1250                             fibManager.removeOrUpdateFibEntry(broker, vpnName, nextHop.getIpAddress(), nh, writeConfigTxn);
1251                         }
1252                     } else {
1253                         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1254                         for (String nh : nhList) {
1255                             //IRT: remove routes from other vpns importing it
1256                             removePrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1257                             for (VpnInstance vpn : vpnsToImportRoute) {
1258                                 String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
1259                                 if (vpnRd != null) {
1260                                     LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd, nextHop.getIpAddress(), vpn.getVpnInstanceName());
1261                                     fibManager.removeOrUpdateFibEntry(broker, vpnRd, nextHop.getIpAddress(), nh, writeConfigTxn);
1262                                 }
1263                             }
1264                         }
1265                     }
1266                     String ip = nextHop.getIpAddress().split("/")[0];
1267                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(broker,
1268                             vpnName, ip);
1269                     if (vpnPortipToPort != null && !vpnPortipToPort.isConfig()) {
1270                         LOG.trace("VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
1271                                 vpnPortipToPort.getPortName(),ip);
1272                         VpnUtil.removeVpnPortFixedIpToPort(broker, vpnName, ip);
1273                     }
1274                 }
1275             }
1276         }
1277     }
1278
1279
1280     private void unbindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName,
1281                                int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval,
1282                                WriteTransaction writeConfigTxn, WriteTransaction writeInvTxn) {
1283         if (!isInterfaceStateDown && isConfigRemoval) {
1284             writeConfigTxn.delete(LogicalDatastoreType.CONFIGURATION,
1285                     InterfaceUtils.buildServiceId(vpnInterfaceName,
1286                             NwConstants.L3VPN_SERVICE_INDEX));
1287         }
1288         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
1289         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
1290                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW, writeInvTxn);
1291         makeArpFlow(dpId, NwConstants.L3VPN_SERVICE_INDEX, lPortTag, vpnInterfaceName,
1292                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.DEL_FLOW, writeInvTxn);
1293     }
1294
1295
1296     private void removePrefixFromBGP(String rd, String prefix, String nextHop, WriteTransaction writeConfigTxn) {
1297         try {
1298             LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
1299             fibManager.removeOrUpdateFibEntry(broker, rd, prefix, nextHop, writeConfigTxn);
1300             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1301             LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
1302         } catch(Exception e) {
1303             LOG.error("Delete prefix failed", e);
1304         }
1305     }
1306
1307     @Override
1308     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
1309         LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
1310         String oldVpnName = original.getVpnInstanceName();
1311         String newVpnName = update.getVpnInstanceName();
1312         BigInteger dpnId = update.getDpnId();
1313         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
1314         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
1315         if (oldAdjs == null) {
1316             oldAdjs = new ArrayList<>();
1317         }
1318         if (newAdjs == null) {
1319             newAdjs = new ArrayList<>();
1320         }
1321         //handles switching between <internal VPN - external VPN>
1322         if (!oldVpnName.equals(newVpnName)) {
1323             remove(identifier, original);
1324             waitForFibToRemoveVpnPrefix(update.getName());
1325             add(identifier, update);
1326         }
1327         //handle both addition and removal of adjacencies
1328         //currently, new adjacency may be an extra route
1329         if (!oldAdjs.equals(newAdjs)) {
1330             for (Adjacency adj : newAdjs) {
1331                 if (oldAdjs.contains(adj)) {
1332                     oldAdjs.remove(adj);
1333                 } else {
1334                     // add new adjacency - right now only extra route will hit this path
1335                     addNewAdjToVpnInterface(identifier, adj, dpnId);
1336                 }
1337             }
1338             for (Adjacency adj : oldAdjs) {
1339                 delAdjFromVpnInterface(identifier, adj, dpnId);
1340             }
1341         }
1342     }
1343
1344     public void processArpRequest(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715
1345                                           .IpAddress srcIP, PhysAddress srcMac, org.opendaylight.yang.gen.v1.urn.ietf
1346             .params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress targetIP, PhysAddress targetMac, String srcInterface){
1347         //Build ARP response with ARP requests TargetIp TargetMac as the Arp Response SrcIp and SrcMac
1348         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
1349                 .setDstIpaddress(srcIP).setDstMacaddress(srcMac).setSrcIpaddress(targetIP).setSrcMacaddress(targetMac).build();
1350         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
1351         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
1352         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
1353             @Override
1354             public void onFailure(Throwable error) {
1355                 LOG.error("Error - {}", msgFormat, error);
1356             }
1357
1358             @Override
1359             public void onSuccess(RpcResult<Void> result) {
1360                 if(!result.isSuccessful()) {
1361                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
1362                 } else {
1363                     LOG.debug("Successful RPC Result - {}", msgFormat);
1364                 }
1365             }
1366         });
1367     }
1368
1369     private String getErrorText(Collection<RpcError> errors) {
1370         StringBuilder errorText = new StringBuilder();
1371         for(RpcError error : errors) {
1372             errorText.append(",").append(error.getErrorType()).append("-")
1373                     .append(error.getMessage());
1374         }
1375         return errorText.toString();
1376     }
1377
1378     private void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1379                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd,
1380                                   WriteTransaction writeOperTxn) {
1381         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1382         Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
1383         Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
1384         Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
1385         if (!isSubnetRoute) {
1386             // NextHop must be present for non-subnetroute entries
1387             Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
1388         }
1389         LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
1390         if (dpnId != null) {
1391             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1392                     .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
1393             LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1394             lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(vpnId)
1395                     .setIsSubnetRoute(isSubnetRoute);
1396             if (elanTag != null) {
1397                 lriBuilder.setElanTag(elanTag);
1398             }
1399             if (vpnInterfaceName != null) {
1400                 lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1401             }
1402             lriBuilder.setParentVpnRd(rd);
1403             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(broker, rd);
1404             if (vpnInstanceOpDataEntry != null) {
1405                 List<String> vpnInstanceNames = Arrays.asList(vpnInstanceOpDataEntry.getVpnInstanceName());
1406                 lriBuilder.setVpnInstanceList(vpnInstanceNames);
1407             }
1408             LabelRouteInfo lri = lriBuilder.build();
1409             LOG.trace("Adding route info to label map: {}", lri);
1410             if (writeOperTxn != null) {
1411                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, lriIid, lri, true);
1412             } else {
1413                 VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
1414             }
1415         } else {
1416             LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
1417         }
1418     }
1419
1420     public synchronized void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
1421                                                         long elantag, BigInteger dpnId, WriteTransaction writeTxn) {
1422         SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
1423         RouteOrigin origin = RouteOrigin.STATIC; // Only case when a route is considered as directly connected
1424         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1425                 .setLabel((long)label).setOrigin(origin.getValue())
1426                 .addAugmentation(SubnetRoute.class, route).build();
1427
1428         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
1429
1430         //TODO: What should be parentVpnId? Get it from RD?
1431         long vpnId = VpnUtil.getVpnId(broker, vpnName);
1432         addToLabelMapper((long)label, dpnId, prefix, Arrays.asList(nextHop), vpnId, null, elantag, true, rd, null);
1433         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1434
1435         InstanceIdentifierBuilder<VrfTables> idBuilder =
1436                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1437         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1438
1439         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1440                 setVrfEntry(vrfEntryList).build();
1441
1442         if (writeTxn != null) {
1443             writeTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1444         } else {
1445             VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1446         }
1447
1448         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1449         if (vpnsToImportRoute.size() > 0) {
1450             VrfEntry importingVrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1451                     .setLabel((long) label).setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
1452                     .addAugmentation(SubnetRoute.class, route).build();
1453             List<VrfEntry> importingVrfEntryList = Arrays.asList(importingVrfEntry);
1454             for (VpnInstance vpnInstance : vpnsToImportRoute) {
1455                 LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop, label, vpnInstance.getVpnInstanceName());
1456                 String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1457                 InstanceIdentifier<VrfTables> importingVrfTableId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(importingRd)).build();
1458                 VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(importingVrfEntryList).build();
1459                 if (writeTxn != null) {
1460                     writeTxn.merge(LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable, true);
1461                 } else {
1462                     VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable);
1463                 }
1464             }
1465         }
1466     }
1467
1468     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1469                                                         SubnetRoute route, WriteTransaction writeConfigTxn) {
1470
1471         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1472         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1473                 .setLabel((long)label).setOrigin(origin.getValue())
1474                 .addAugmentation(SubnetRoute.class, route).build();
1475         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, route.getElantag());
1476         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1477         InstanceIdentifierBuilder<VrfTables> idBuilder =
1478                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1479         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1480         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1481                 setVrfEntry(vrfEntryList).build();
1482         if (writeConfigTxn != null) {
1483             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1484         } else {
1485             VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1486         }
1487     }
1488
1489     public synchronized void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName){
1490         fibManager.removeOrUpdateFibEntry(broker, rd, prefix, null /* nextHopToRemove */, null);
1491         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1492         for (VpnInstance vpnInstance : vpnsToImportRoute) {
1493             String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1494             LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix, vpnInstance.getVpnInstanceName());
1495             fibManager.removeOrUpdateFibEntry(broker, importingRd, prefix, null /* nextHopToRemove */, null);
1496         }
1497     }
1498
1499     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1500
1501         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
1502
1503         if (optVpnInterface.isPresent()) {
1504             VpnInterface currVpnIntf = optVpnInterface.get();
1505             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1506             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1507             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1508             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, adjPath);
1509             long label =
1510                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1511                             VpnUtil.getNextHopLabelKey((rd != null) ? rd : currVpnIntf.getVpnInstanceName(), prefix));
1512
1513             List<Adjacency> adjacencies;
1514             if (optAdjacencies.isPresent()) {
1515                 adjacencies = optAdjacencies.get().getAdjacency();
1516             } else {
1517                 //This code will not be hit since VM adjacency will always be there
1518                 adjacencies = new ArrayList<>();
1519             }
1520
1521             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIpList(adj.getNextHopIpList())
1522                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
1523
1524             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1525             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove());
1526
1527             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1528             for (String nh : adj.getNextHopIpList()) {
1529                 addExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(), (int) label,
1530                         currVpnIntf.getName());
1531             }
1532         }
1533     }
1534
1535     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1536         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
1537
1538         if (optVpnInterface.isPresent()) {
1539             VpnInterface currVpnIntf = optVpnInterface.get();
1540
1541             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1542             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
1543             if (optAdjacencies.isPresent()) {
1544                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1545
1546                 if (!adjacencies.isEmpty()) {
1547                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1548                     LOG.trace("Adjacencies are " + adjacencies);
1549                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1550                     while (adjIt.hasNext()) {
1551                         Adjacency adjElem = adjIt.next();
1552                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1553                             adjIt.remove();
1554
1555                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1556                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1557                                     currVpnIntf.getVpnInstanceName(),
1558                                     aug, dpnId, currVpnIntf.isScheduledForRemove());
1559
1560                             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1561
1562                             for (String nh : adj.getNextHopIpList()) {
1563                                 delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
1564                                         currVpnIntf.getName());
1565                             }
1566                             break;
1567                         }
1568
1569                     }
1570                 }
1571             }
1572         }
1573
1574     }
1575
1576     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label,
1577                                  String intfName) {
1578
1579         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1580         VpnUtil.syncUpdate(
1581                 broker,
1582                 LogicalDatastoreType.OPERATIONAL,
1583                 VpnUtil.getVpnToExtrarouteIdentifier( (rd != null) ? rd : routerID, destination),
1584                 VpnUtil.getVpnToExtraroute(destination, Arrays.asList(nextHop)));
1585
1586         BigInteger dpnId = null;
1587         if (intfName != null && !intfName.isEmpty()) {
1588             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1589             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
1590             if (nextHopIp == null || nextHopIp.isEmpty()) {
1591                 LOG.error("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1592                         intfName, destination);
1593                 return;
1594             }
1595             nextHop = nextHopIp;
1596         }
1597         List<String> nextHopIpList = Arrays.asList(nextHop);
1598         if (rd != null) {
1599             /* Label mapper is required only for BGP VPN and not for Internal VPN */
1600             addToLabelMapper((long) label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(broker, routerID),
1601                     intfName, null, false, rd, null);
1602         }
1603
1604         // TODO (eperefr): This is a limitation to be stated in docs. When configuring static route to go to
1605         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1606         // first place.
1607         InterVpnLink interVpnLink = VpnUtil.getInterVpnLinkByEndpointIp(broker, nextHop);
1608         if ( interVpnLink != null ) {
1609             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1610             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1611             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1612             // This is like leaking one of the Vpn2 routes towards Vpn1
1613             boolean nexthopIsVpn2 = ( interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nextHop) );
1614             String srcVpnUuid = (nexthopIsVpn2) ? interVpnLink.getSecondEndpoint().getVpnUuid().getValue()
1615                     : interVpnLink.getFirstEndpoint().getVpnUuid().getValue();
1616             String dstVpnUuid = (nexthopIsVpn2) ? interVpnLink.getFirstEndpoint().getVpnUuid().getValue()
1617                     : interVpnLink.getSecondEndpoint().getVpnUuid().getValue();
1618             String dstVpnRd = VpnUtil.getVpnRd(broker, dstVpnUuid);
1619             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1620                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1621             VpnUtil.leakRoute(broker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel);
1622         } else {
1623             if (rd != null) {
1624                 addPrefixToBGP(rd, destination, nextHop, label, null);
1625             } else {
1626                 // ### add FIB route directly
1627                 fibManager.addOrUpdateFibEntry(broker, routerID, destination, Arrays.asList(nextHop), label, RouteOrigin.STATIC, null);
1628             }
1629         }
1630     }
1631
1632     protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName) {
1633         if (intfName != null && !intfName.isEmpty()) {
1634             BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1635             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
1636             if (nextHopIp == null || nextHopIp.isEmpty()) {
1637                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1638                         intfName, destination);
1639             }
1640             nextHop = nextHopIp;
1641         }
1642
1643         if (rd != null) {
1644             removePrefixFromBGP(rd, destination, nextHop, null);
1645         } else {
1646             // ### add FIB route directly
1647             fibManager.removeOrUpdateFibEntry(broker, routerID, destination, nextHop, null);
1648         }
1649     }
1650
1651     class VpnInterfaceOpListener extends org.opendaylight.genius.mdsalutil.AbstractDataChangeListener<VpnInterface> {
1652
1653         public VpnInterfaceOpListener() {
1654             super(VpnInterface.class);
1655         }
1656
1657         @Override
1658         protected void remove(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface del) {
1659             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1660             final String interfaceName = key.getName();
1661             final String vpnName = del.getVpnInstanceName();
1662             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1663             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1664                     new Callable<List<ListenableFuture<Void>>>() {
1665                         @Override
1666                         public List<ListenableFuture<Void>> call() throws Exception {
1667                             WriteTransaction writeOperTxn = broker.newWriteOnlyTransaction();
1668                             postProcessVpnInterfaceRemoval(identifier, del, writeOperTxn);
1669                             CheckedFuture<Void, TransactionCommitFailedException> futures = writeOperTxn.submit();
1670                             try {
1671                                 futures.get();
1672                             } catch (InterruptedException | ExecutionException e) {
1673                                 LOG.error("Error handling removal of oper data for interface {} from vpn {}", interfaceName,
1674                                         vpnName);
1675                                 throw new RuntimeException(e.getMessage());
1676                             }
1677                             return null;
1678                         }
1679                     });
1680         }
1681
1682         private void postProcessVpnInterfaceRemoval(InstanceIdentifier<VpnInterface> identifier, VpnInterface del,
1683                                                     WriteTransaction writeOperTxn) {
1684             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1685             String interfaceName = key.getName();
1686             String vpnName = del.getVpnInstanceName();
1687
1688             LOG.trace("VpnInterfaceOpListener removed: interface name {} vpnName {}", interfaceName, vpnName);
1689             //decrement the vpn interface count in Vpn Instance Op Data
1690             InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to
1691                     .vpn.id.VpnInstance>
1692                     id = VpnUtil.getVpnInstanceToVpnIdIdentifier(vpnName);
1693             Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id
1694                     .VpnInstance> vpnInstance
1695                     = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
1696
1697             if (vpnInstance.isPresent()) {
1698                 String rd = null;
1699                 rd = vpnInstance.get().getVrfId();
1700                 //String rd = getRouteDistinguisher(del.getVpnInstanceName());
1701
1702                 VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
1703                 LOG.trace("VpnInterfaceOpListener removed: interface name {} rd {} vpnName {} in Vpn Op Instance {}",
1704                         interfaceName, rd, vpnName, vpnInstOp);
1705
1706                 if (vpnInstOp != null) {
1707                     // Vpn Interface removed => No more adjacencies from it.
1708                     // Hence clean up interface from vpn-dpn-interface list.
1709                     Adjacency adjacency = del.getAugmentation(Adjacencies.class).getAdjacency().get(0);
1710                     List<Prefixes> prefixToInterface = new ArrayList<>();
1711                     Optional<Prefixes> prefix = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1712                             VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1713                                     VpnUtil.getIpPrefix(adjacency.getIpAddress())));
1714                     if (prefix.isPresent()) {
1715                         prefixToInterface.add(prefix.get());
1716                     }
1717                     if (prefixToInterface.isEmpty()) {
1718                         for (String nh : adjacency.getNextHopIpList()) {
1719                             prefix = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1720                                     VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1721                                             VpnUtil.getIpPrefix(nh)));
1722                             if (prefix.isPresent())
1723                                 prefixToInterface.add(prefix.get());
1724                         }
1725                     }
1726                     for (Prefixes pref : prefixToInterface) {
1727                         if (writeOperTxn != null) {
1728                             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1729                                     VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(), pref.getIpAddress()));
1730                         } else {
1731                             VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
1732                                     VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(), pref.getIpAddress()),
1733                                     VpnUtil.DEFAULT_CALLBACK);
1734                         }
1735                         updateVpnToDpnMapping(pref.getDpnId(), del.getVpnInstanceName(), interfaceName, false /* delete */);
1736                     }
1737                 }
1738             } else {
1739                 LOG.error("rd not retrievable as vpninstancetovpnid for vpn {} is absent, trying rd as ", vpnName, vpnName);
1740             }
1741             notifyTaskIfRequired(interfaceName);
1742         }
1743
1744         private void notifyTaskIfRequired(String intfName) {
1745             Runnable notifyTask = vpnIntfMap.remove(intfName);
1746             if (notifyTask == null) {
1747                 LOG.trace("VpnInterfaceOpListener update: No Notify Task queued for vpnInterface {}", intfName);
1748                 return;
1749             }
1750             executorService.execute(notifyTask);
1751         }
1752
1753         @Override
1754         protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1755                               final VpnInterface update) {
1756             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1757             final String interfaceName = key.getName();
1758
1759             if (original.getVpnInstanceName().equals(update.getVpnInstanceName())) {
1760                 return;
1761             }
1762
1763             final String vpnName = update.getVpnInstanceName();
1764             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1765             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1766                     new Callable<List<ListenableFuture<Void>>>() {
1767                         @Override
1768                         public List<ListenableFuture<Void>> call() throws Exception {
1769                             postProcessVpnInterfaceUpdate(identifier, original, update);
1770                             return null;
1771                         }
1772                     });
1773         }
1774
1775         private void postProcessVpnInterfaceUpdate(InstanceIdentifier<VpnInterface> identifier, VpnInterface original,
1776                                                    VpnInterface update) {
1777             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1778             String interfaceName = key.getName();
1779
1780             //increment the vpn interface count in Vpn Instance Op Data
1781             VpnInstanceOpDataEntry vpnInstOp = null;
1782             InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to
1783                     .vpn.id.VpnInstance>
1784                     origId = VpnUtil.getVpnInstanceToVpnIdIdentifier(original.getVpnInstanceName());
1785             Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id
1786                     .VpnInstance> origVpnInstance
1787                     = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, origId);
1788
1789             if (origVpnInstance.isPresent()) {
1790                 String rd = null;
1791                 rd = origVpnInstance.get().getVrfId();
1792
1793                 vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
1794                 LOG.trace("VpnInterfaceOpListener updated: interface name {} original rd {} original vpnName {}",
1795                         interfaceName, rd, original.getVpnInstanceName());
1796
1797                 if (vpnInstOp != null) {
1798                     Adjacency adjacency = original.getAugmentation(Adjacencies.class).getAdjacency().get(0);
1799                     List<Prefixes> prefixToInterfaceList = new ArrayList<>();
1800                     Optional<Prefixes> prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1801                             VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1802                                     VpnUtil.getIpPrefix(adjacency.getIpAddress())));
1803                     if (prefixToInterface.isPresent()) {
1804                         prefixToInterfaceList.add(prefixToInterface.get());
1805                     } else {
1806                         for (String adj : adjacency.getNextHopIpList()) {
1807                             prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1808                                     VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1809                                             VpnUtil.getIpPrefix(adj)));
1810                             if (prefixToInterface.isPresent()) {
1811                                 prefixToInterfaceList.add(prefixToInterface.get());
1812                             }
1813                         }
1814                     }
1815                     for (Prefixes prefix : prefixToInterfaceList) {
1816                         updateVpnToDpnMapping(prefix.getDpnId(), original.getVpnInstanceName(), interfaceName, false /* delete */);
1817                     }
1818                 }
1819             }
1820             notifyTaskIfRequired(interfaceName);
1821         }
1822
1823         @Override
1824         protected void add(InstanceIdentifier<VpnInterface> identifier, VpnInterface add) {
1825         }
1826     }
1827
1828     void publishAddNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1829         LOG.debug("Sending notification for add dpn {} in vpn {} event ", dpnId, vpnName);
1830         AddEventData data = new AddEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1831         AddDpnEvent event = new AddDpnEventBuilder().setAddEventData(data).build();
1832         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1833         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1834             @Override
1835             public void onFailure(Throwable error) {
1836                 LOG.warn("Error in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName, error);
1837             }
1838
1839             @Override
1840             public void onSuccess(Object arg) {
1841                 LOG.trace("Successful in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName);
1842             }
1843         });
1844     }
1845
1846     void publishRemoveNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1847         LOG.debug("Sending notification for remove dpn {} in vpn {} event ", dpnId, vpnName);
1848         RemoveEventData data = new RemoveEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1849         RemoveDpnEvent event = new RemoveDpnEventBuilder().setRemoveEventData(data).build();
1850         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1851         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1852             @Override
1853             public void onFailure(Throwable error) {
1854                 LOG.warn("Error in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName, error);
1855             }
1856
1857             @Override
1858             public void onSuccess(Object arg) {
1859                 LOG.trace("Successful in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName);
1860             }
1861         });
1862     }
1863
1864     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1865         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1866                 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1867                 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1868     }
1869
1870     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1871         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1872                 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1873     }
1874
1875     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1876         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1877         if(dpId.equals(BigInteger.ZERO)) {
1878             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
1879             return;
1880         }
1881         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1882
1883         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
1884                 .OPERATIONAL, routerDpnListIdentifier);
1885         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1886         if (optionalRouterDpnList.isPresent()) {
1887             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1888                     RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1889         } else {
1890             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1891             builder.setRouterId(routerName);
1892             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1893             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
1894             routerInterfaces.add(routerInterface);
1895             builder.setDpnVpninterfacesList(Arrays.asList(dpnVpnList.build()));
1896             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1897                     getRouterId(routerName),
1898                     builder.build(), true);
1899         }
1900     }
1901
1902     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeOperTxn) {
1903         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1904         if(dpId.equals(BigInteger.ZERO)) {
1905             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1906             return;
1907         }
1908         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1909         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
1910                 .OPERATIONAL, routerDpnListIdentifier);
1911         if (optionalRouterDpnList.isPresent()) {
1912             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1913             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1914
1915             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1916                 if (routerInterfaces.isEmpty()) {
1917                     if (writeOperTxn != null) {
1918                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1919                     } else {
1920                         MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1921                     }
1922                 } else {
1923                     if (writeOperTxn != null) {
1924                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1925                                 RouterInterfaces.class,
1926                                 new RouterInterfacesKey(vpnInterfaceName)));
1927                     } else {
1928                         MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1929                                 RouterInterfaces.class,
1930                                 new RouterInterfacesKey(vpnInterfaceName)));
1931                     }
1932                 }
1933             }
1934         }
1935     }
1936
1937     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
1938                                                   WriteTransaction writeOperTxn) {
1939         if(dpId.equals(BigInteger.ZERO)) {
1940             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1941             return;
1942         }
1943         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1944         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
1945                 .OPERATIONAL, routerDpnListIdentifier);
1946         if (optionalRouterDpnList.isPresent()) {
1947             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1948             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1949             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1950                 if (routerInterfaces.isEmpty()) {
1951                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
1952                 } else {
1953                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1954                             RouterInterfaces.class,
1955                             new RouterInterfacesKey(vpnInterfaceName)));
1956                 }
1957             }
1958         }
1959     }
1960
1961     public void addMIPAdjacency(String vpnName,String vpnInterface, org.opendaylight.yang.gen.v1.urn.ietf.params.xml
1962             .ns.yang.ietf.inet.types.rev130715.IpAddress prefix){
1963
1964         LOG.trace("Adding {} adjacency to VPN Interface {} ",prefix,vpnInterface);
1965         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
1966         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
1967         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
1968         String nextHopIpAddr = null;
1969         String nextHopMacAddress = null;
1970         String ip = prefix.getIpv4Address().getValue();
1971         if (adjacencies.isPresent()) {
1972             List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
1973             ip = VpnUtil.getIpPrefix(ip);
1974             for (Adjacency adjacs : adjacencyList) {
1975                 if (adjacs.getMacAddress() != null && !adjacs.getMacAddress().isEmpty()) {
1976                     nextHopIpAddr = adjacs.getIpAddress();
1977                     nextHopMacAddress = adjacs.getMacAddress();
1978                     break;
1979                 }
1980             }
1981             if (nextHopMacAddress != null && ip != null) {
1982                 String rd = VpnUtil.getVpnRd(broker, vpnName);
1983                 long label =
1984                         VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1985                                 VpnUtil.getNextHopLabelKey((rd != null) ? rd : vpnName, ip));
1986                 String nextHopIp = nextHopIpAddr.split("/")[0];
1987                 Adjacency newAdj = new AdjacencyBuilder().setIpAddress(ip).setKey
1988                         (new AdjacencyKey(ip)).setNextHopIpList(Arrays.asList(nextHopIp)).build();
1989                 adjacencyList.add(newAdj);
1990                 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
1991                 VpnInterface newVpnIntf = new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(vpnInterface)).
1992                         setName(vpnInterface).setVpnInstanceName(vpnName).addAugmentation(Adjacencies.class, aug).build();
1993                 VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vpnIfId, newVpnIntf);
1994                 LOG.debug(" Successfully stored subnetroute Adjacency into VpnInterface {}", vpnInterface);
1995             }
1996         }
1997
1998     }
1999
2000     public void removeMIPAdjacency(String vpnName, String vpnInterface, org.opendaylight.yang.gen.v1.urn.ietf.params
2001             .xml.ns.yang.ietf.inet.types.rev130715.IpAddress prefix) {
2002         String ip = VpnUtil.getIpPrefix(prefix.getIpv4Address().getValue());
2003         LOG.trace("Removing {} adjacency from Old VPN Interface {} ",ip,vpnInterface);
2004         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
2005         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
2006         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
2007         if (adjacencies.isPresent()) {
2008             InstanceIdentifier<Adjacency> adjacencyIdentifier = InstanceIdentifier.builder(VpnInterfaces.class).
2009                     child(VpnInterface.class, new VpnInterfaceKey(vpnInterface)).augmentation(Adjacencies.class)
2010                     .child(Adjacency.class, new AdjacencyKey(ip)).build();
2011             MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, adjacencyIdentifier);
2012             LOG.trace("Successfully Deleted Adjacency into VpnInterface {}", vpnInterface);
2013         }
2014     }
2015 }