vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.*;
  11 import com.google.common.collect.FluentIterable;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.ListenableFuture;
  14 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  15 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  16 import org.opendaylight.netvirt.bgpmanager.api.RouteOrigin;
  17 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  18 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  19
  20 import com.google.common.util.concurrent.FutureCallback;
  21 import com.google.common.util.concurrent.Futures;
  22 import com.google.common.util.concurrent.JdkFutureAdapters;
  23
  24 import org.opendaylight.controller.md.sal.binding.api.*;
  25 import org.opendaylight.genius.mdsalutil.*;
  26 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  27 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.VpnTargets;
  28 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  29 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  30 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeExternal;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeHwvtep;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeInternal;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.IsDcgwPresentInputBuilder;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.IsDcgwPresentOutput;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.*;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventData;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventDataBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventData;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventDataBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  74
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
  76         .VpnInstanceOpDataEntryBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInput;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpResponseInputBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  94
  95 import java.math.BigInteger;
  96 import java.util.Collection;
  97 import java.util.List;
  98 import java.util.ArrayList;
  99 import java.util.Arrays;
 100 import java.util.Iterator;
 101 import java.util.concurrent.*;
 102
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 106 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 110 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 111 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 112 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 113 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 114 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
 115 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 118 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
 119 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
 120 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
 121 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
 122 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
 123 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
 124 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
 125 import org.opendaylight.yangtools.yang.common.RpcError;
 126 import org.opendaylight.yangtools.yang.common.RpcResult;
 127 import org.slf4j.Logger;
 128 import org.slf4j.LoggerFactory;
 129
 130 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
 131     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 132     private ListenerRegistration<DataChangeListener> listenerRegistration, opListenerRegistration, tunnelInterfaceStateListenerRegistration;
 133     private final DataBroker broker;
 134     private final IBgpManager bgpManager;
 135     private IFibManager fibManager;
 136     private IMdsalApiManager mdsalManager;
 137     private OdlInterfaceRpcService ifaceMgrRpcService;
 138     private ItmRpcService itmProvider;
 139     private IdManagerService idManager;
 140     private OdlArputilService arpManager;
 141     private NeutronvpnService neuService;
 142     private VpnSubnetRouteHandler vpnSubnetRouteHandler;
 143     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
 144     private ExecutorService executorService = Executors.newSingleThreadExecutor();
 145     private InterfaceStateChangeListener interfaceListener;
 146     private SubnetRouteInterfaceStateChangeListener subnetRouteInterfaceListener;
 147     private TunnelInterfaceStateListener tunnelInterfaceStateListener;
 148     private VpnInterfaceOpListener vpnInterfaceOpListener;
 149     private ArpNotificationHandler arpNotificationHandler;
 150     private DpnInVpnChangeListener dpnInVpnChangeListener;
 151     private NotificationPublishService notificationPublishService;
 152     private FibRpcService fibService;
 153     protected enum UpdateRouteAction {
 154         ADVERTISE_ROUTE, WITHDRAW_ROUTE
 155     }
 156
 157     /**
 158      * Responsible for listening to data change related to VPN Interface
 159      * Bind VPN Service on the interface and informs the BGP service
 160      *
 161      * @param db - dataBroker service reference
 162      * @param bgpManager Used to advertise routes to the BGP Router
 163      * @param notificationService Used to subscribe to notification events
 164      */
 165     public VpnInterfaceManager(final DataBroker db, final IBgpManager bgpManager, NotificationService notificationService) {
 166         super(VpnInterface.class);
 167         broker = db;
 168         this.bgpManager = bgpManager;
 169         interfaceListener = new InterfaceStateChangeListener(db, this);
 170         subnetRouteInterfaceListener = new SubnetRouteInterfaceStateChangeListener(db, this);
 171         vpnInterfaceOpListener = new VpnInterfaceOpListener();
 172         arpNotificationHandler = new ArpNotificationHandler(this, broker);
 173         vpnSubnetRouteHandler = new VpnSubnetRouteHandler(broker, bgpManager, this);
 174         dpnInVpnChangeListener = new DpnInVpnChangeListener(broker);
 175         tunnelInterfaceStateListener = new TunnelInterfaceStateListener(broker, this);
 176         notificationService.registerNotificationListener(vpnSubnetRouteHandler);
 177         notificationService.registerNotificationListener(arpNotificationHandler);
 178         notificationService.registerNotificationListener(dpnInVpnChangeListener);
 179         registerListener(db);
 180     }
 181
 182     public void setMdsalManager(IMdsalApiManager mdsalManager) {
 183         this.mdsalManager = mdsalManager;
 184     }
 185
 186     public void setIfaceMgrRpcService(OdlInterfaceRpcService ifMgrRpcService) {
 187         this.ifaceMgrRpcService = ifMgrRpcService;
 188         interfaceListener.setIfaceMgrRpcService(ifMgrRpcService);
 189     }
 190
 191     public void setITMProvider(ItmRpcService itmProvider) {
 192         this.itmProvider = itmProvider;
 193     }
 194
 195
 196     public void setFibManager(IFibManager fibManager) {
 197         this.fibManager = fibManager;
 198     }
 199
 200     public IFibManager getFibManager() {
 201         return this.fibManager;
 202     }
 203
 204
 205     public void setIdManager(IdManagerService idManager) {
 206         this.idManager = idManager;
 207         vpnSubnetRouteHandler.setIdManager(idManager);
 208     }
 209
 210     public void setArpManager(OdlArputilService arpManager) {
 211         this.arpManager = arpManager;
 212     }
 213
 214     void setNotificationPublishService(NotificationPublishService notificationPublishService) {
 215         this.notificationPublishService = notificationPublishService;
 216     }
 217
 218     public void setNeutronvpnManager(NeutronvpnService neuService) { this.neuService = neuService; }
 219
 220     public void setFibRpcService(FibRpcService fibService) {
 221         this.fibService = fibService;
 222     }
 223
 224     public FibRpcService getFibRpcService() {
 225         return fibService;
 226     }
 227
 228     public VpnSubnetRouteHandler getVpnSubnetRouteHandler() {
 229         return this.vpnSubnetRouteHandler;
 230     }
 231
 232     @Override
 233     public void close() throws Exception {
 234         if (listenerRegistration != null) {
 235             try {
 236                 listenerRegistration.close();
 237                 opListenerRegistration.close();
 238             } catch (final Exception e) {
 239                 LOG.error("Error when cleaning up DataChangeListener.", e);
 240             }
 241             listenerRegistration = null;
 242             opListenerRegistration = null;
 243         }
 244         LOG.info("VPN Interface Manager Closed");
 245     }
 246
 247     private void registerListener(final DataBroker db) {
 248         try {
 249             listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 250                     getWildCardPath(), VpnInterfaceManager.this, DataChangeScope.SUBTREE);
 251             opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
 252                                                                    getWildCardPath(), vpnInterfaceOpListener, DataChangeScope.SUBTREE);
 253             tunnelInterfaceStateListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
 254                     getTunnelInterfaceStateListenerPath(), tunnelInterfaceStateListener, DataChangeScope.SUBTREE);
 255         } catch (final Exception e) {
 256             LOG.error("VPN Service DataChange listener registration fail!", e);
 257             throw new IllegalStateException("VPN Service registration Listener failed.", e);
 258         }
 259     }
 260
 261     private InstanceIdentifier<StateTunnelList> getTunnelInterfaceStateListenerPath() {
 262         return InstanceIdentifier.create(TunnelsState.class).child(StateTunnelList.class);
 263     }
 264
 265     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 266         return InstanceIdentifier.create(InterfacesState.class)
 267             .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 268     }
 269
 270     @Override
 271     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 272         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 273         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 274         String interfaceName = key.getName();
 275
 276         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 277             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 278         if(interfaceState != null){
 279             try{
 280                 final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 281                 final int ifIndex = interfaceState.getIfIndex();
 282                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 283                 dataStoreCoordinator.enqueueJob("VPNINTERFACE-"+ interfaceName,
 284                         new Callable<List<ListenableFuture<Void>>>() {
 285                             @Override
 286                             public List<ListenableFuture<Void>> call() throws Exception {
 287                                 WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
 288                                 processVpnInterfaceUp(dpnId, vpnInterface, ifIndex, false, writeTxn);
 289                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 290                                 futures.add(writeTxn.submit());
 291                                 return futures;
 292                             }
 293                         });
 294             }catch (Exception e){
 295                 LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", interfaceName, e);
 296                 return;
 297             }
 298         } else {
 299             LOG.info("Handling addition of VPN interface {} skipped as interfaceState is not available", interfaceName);
 300         }
 301     }
 302
 303     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface,
 304                                          final int lPortTag, boolean isInterfaceUp, WriteTransaction writeTxn) {
 305
 306         final String interfaceName = vpnInterface.getName();
 307         if (!isInterfaceUp) {
 308             final String vpnName = vpnInterface.getVpnInstanceName();
 309             LOG.info("Binding vpn service to interface {} ", interfaceName);
 310             long vpnId = VpnUtil.getVpnId(broker, vpnName);
 311             if (vpnId == VpnConstants.INVALID_ID) {
 312                 LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 313                 return;
 314             }
 315             boolean waitForVpnInterfaceOpRemoval = false;
 316             int numAdjs = 0;
 317             VpnInterface opVpnInterface = null;
 318             synchronized (interfaceName.intern()) {
 319                 opVpnInterface = VpnUtil.getOperationalVpnInterface(broker, vpnInterface.getName());
 320                 if (opVpnInterface != null ) {
 321                     String opVpnName = opVpnInterface.getVpnInstanceName();
 322                     String primaryInterfaceIp = null;
 323                     if(opVpnName.equals(vpnName)) {
 324                         // Please check if the primary VRF Entry does not exist for VPNInterface
 325                         // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 326                         // back to back
 327                         // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 328                         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(broker, interfaceName);
 329                         if (adjs == null) {
 330                             LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained", interfaceName);
 331                             return;
 332                         }
 333                         numAdjs = adjs.size();
 334                         for (Adjacency adj : adjs) {
 335                             if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
 336                                 primaryInterfaceIp = adj.getIpAddress();
 337                                 break;
 338                             }
 339                         }
 340                         if (primaryInterfaceIp == null) {
 341                             LOG.info("VPN Interface {} addition failed as primary adjacency for this vpn interface could not be obtained", interfaceName);
 342                             return;
 343                         }
 344                         // Get the rd of the vpn instance
 345                         String rd = getRouteDistinguisher(opVpnName);
 346                         VrfEntry vrf = VpnUtil.getVrfEntry(broker, rd, primaryInterfaceIp);
 347                             if (vrf != null) {
 348                                 LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
 349                                 return;
 350                             }
 351                         waitForVpnInterfaceOpRemoval = true;
 352                     } else {
 353                         LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
 354                              interfaceName, vpnName, opVpnName);
 355                     }
 356                 }
 357                 if (!waitForVpnInterfaceOpRemoval) {
 358                     // Add the VPNInterface and quit
 359                     DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 360                     dataStoreCoordinator.enqueueJob((interfaceName + dpId.toString()),
 361                             new UpdateDpnToVpnWorker(dpId, vpnName, interfaceName, lPortTag, true));
 362                     return;
 363                 }
 364             }
 365
 366             // FIB didn't get a chance yet to clean up this VPNInterface
 367             // Let us give it a chance here !
 368             LOG.info("VPN Interface {} waiting for FIB to clean up! ", interfaceName);
 369             try {
 370                 Runnable notifyTask = new VpnNotifyTask();
 371                 vpnIntfMap.put(interfaceName, notifyTask);
 372                 synchronized (notifyTask) {
 373                     try {
 374                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 375                     } catch (InterruptedException e) {
 376                     }
 377                 }
 378             } finally {
 379                 vpnIntfMap.remove(interfaceName);
 380             }
 381
 382             opVpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
 383             if (opVpnInterface != null) {
 384                 LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
 385                 return;
 386             }
 387             // VPNInterface got removed, proceed with Add
 388             synchronized (interfaceName.intern()) {
 389                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 390                 dataStoreCoordinator.enqueueJob((interfaceName + dpId.toString()),
 391                         new UpdateDpnToVpnWorker(dpId, vpnName, interfaceName, lPortTag, true));
 392             }
 393         } else {
 394             synchronized (interfaceName.intern()) {
 395                 // Interface is retained in the DPN, but its Link Up.
 396                 // Advertise prefixes again for this interface to BGP
 397                 advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
 398                         vpnInterface, writeTxn);
 399             }
 400         }
 401     }
 402
 403
 404     private class UpdateDpnToVpnWorker implements Callable<List<ListenableFuture<Void>>> {
 405         BigInteger dpnId;
 406         String vpnName;
 407         String interfaceName;
 408         boolean addToDpn;
 409         int lPortTag;
 410
 411         public UpdateDpnToVpnWorker(BigInteger dpnId, String vpnName, String interfaceName,
 412                                     int lPortTag, boolean addToDpn) {
 413             this.dpnId= dpnId;
 414             this.vpnName = vpnName;
 415             this.interfaceName = interfaceName;
 416             this.lPortTag = lPortTag;
 417             this.addToDpn = addToDpn;
 418         }
 419
 420         @Override
 421         public List<ListenableFuture<Void>> call() throws Exception {
 422             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
 423             // to call the respective helpers.
 424             WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
 425             updateDpnDbs(dpnId, vpnName, interfaceName, addToDpn, writeTxn);
 426             List<ListenableFuture<Void>> futures = new ArrayList<>();
 427             futures.add(writeTxn.submit());
 428             ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
 429             Futures.addCallback(listenableFuture,
 430                     new UpdateDpnToVpnCallback(dpnId, vpnName, interfaceName, lPortTag, addToDpn));
 431             return futures;
 432         }
 433     }
 434
 435
 436     /**
 437      * JobCallback class is used as a future callback for
 438      * main and rollback workers to handle success and failure.
 439      */
 440     private class UpdateDpnToVpnCallback implements FutureCallback<List<Void>> {
 441         BigInteger dpnId;
 442         String vpnName;
 443         String interfaceName;
 444         boolean addToDpn;
 445         int lPortTag;
 446
 447         public UpdateDpnToVpnCallback(BigInteger dpnId, String vpnName, String interfaceName,
 448                                       int lPortTag, boolean addToDpn) {
 449             this.dpnId= dpnId;
 450             this.vpnName = vpnName;
 451             this.interfaceName = interfaceName;
 452             this.lPortTag = lPortTag;
 453             this.addToDpn = addToDpn;
 454         }
 455
 456         /**
 457          * @param voids
 458          * This implies that all the future instances have returned success. -- TODO: Confirm this
 459          */
 460         @Override
 461         public void onSuccess(List<Void> voids) {
 462             WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
 463             bindService(dpnId, vpnName, interfaceName, lPortTag, writeTxn);
 464             processVpnInterfaceAdjacencies(dpnId, vpnName, interfaceName, writeTxn);
 465             writeTxn.submit();
 466         }
 467
 468         /**
 469          *
 470          * @param throwable
 471          * This method is used to handle failure callbacks.
 472          * If more retry needed, the retrycount is decremented and mainworker is executed again.
 473          * After retries completed, rollbackworker is executed.
 474          * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
 475          */
 476
 477         @Override
 478         public void onFailure(Throwable throwable) {
 479             LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
 480         }
 481     }
 482
 483
 484     private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
 485                                                  VpnInterface intf, WriteTransaction writeTxn) {
 486         //Read NextHops
 487         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 488         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 489
 490         String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
 491         if (rd == null) {
 492             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 493                     intf.getName(), intf.getVpnInstanceName());
 494             return;
 495         } else {
 496             if (rd.equals(intf.getVpnInstanceName())) {
 497                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in " +
 498                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 499
 500                 return;
 501             }
 502         }
 503         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
 504                 intf.getVpnInstanceName(), rd);
 505
 506         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 507         if (nextHopIp == null){
 508             LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
 509             return;
 510         }
 511
 512         if (adjacencies.isPresent()) {
 513             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 514
 515             if (!nextHops.isEmpty()) {
 516                 LOG.trace("NextHops are " + nextHops);
 517                 for (Adjacency nextHop : nextHops) {
 518                     long label = nextHop.getLabel();
 519                     try {
 520                         LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 521                         bgpManager.advertisePrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
 522                         LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 523                     } catch(Exception e) {
 524                         LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
 525                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 526                     }
 527                 }
 528             }
 529         }
 530     }
 531
 532     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 533         //Read NextHops
 534         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 535         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 536
 537         String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
 538         if (rd == null) {
 539             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 540                     intf.getName(), intf.getVpnInstanceName());
 541             return;
 542         } else {
 543             if (rd.equals(intf.getVpnInstanceName())) {
 544                 LOG.info("withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in " +
 545                         "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
 546                 return;
 547             }
 548         }
 549         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
 550                 intf.getVpnInstanceName(), rd);
 551         if (adjacencies.isPresent()) {
 552             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 553
 554             if (!nextHops.isEmpty()) {
 555                 LOG.trace("NextHops are " + nextHops);
 556                 for (Adjacency nextHop : nextHops) {
 557                     try {
 558                         LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 559                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 560                         LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
 561                     } catch(Exception e) {
 562                         LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
 563                                 nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
 564                     }
 565                 }
 566             }
 567         }
 568     }
 569
 570     private void updateDpnDbs(BigInteger dpId, String vpnName, String interfaceName, boolean add, WriteTransaction writeTxn) {
 571         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 572         if (dpId == null) {
 573             dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, interfaceName);
 574         }
 575         if(!dpId.equals(BigInteger.ZERO)) {
 576             if(add)
 577                 updateMappingDbs(vpnId, dpId, interfaceName, vpnName, writeTxn);
 578             else
 579                 removeFromMappingDbs(vpnId, dpId, interfaceName, vpnName, writeTxn);
 580         }
 581     }
 582
 583     private void bindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName, int lPortTag,
 584                              WriteTransaction writeTxn) {
 585         int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 586         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 587
 588         int instructionKey = 0;
 589         List<Instruction> instructions = new ArrayList<Instruction>();
 590
 591         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 592         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 593
 594         BoundServices
 595             serviceInfo =
 596             InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 597                                             VpnConstants.L3VPN_SERVICE_IDENTIFIER, priority,
 598                                             VpnConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 599         writeTxn.put(LogicalDatastoreType.CONFIGURATION,
 600                 InterfaceUtils.buildServiceId(vpnInterfaceName, VpnConstants.L3VPN_SERVICE_IDENTIFIER), serviceInfo, true);
 601         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 602                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW, writeTxn);
 603         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 604                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.ADD_FLOW, writeTxn);
 605
 606     }
 607
 608     private void processVpnInterfaceAdjacencies(BigInteger dpnId, String vpnName, String interfaceName,
 609                                                 WriteTransaction writeTxn) {
 610         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 611
 612         synchronized (interfaceName) {
 613             // Read NextHops
 614             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 615             Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
 616
 617             if (adjacencies.isPresent()) {
 618                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 619                 List<Adjacency> value = new ArrayList<>();
 620
 621                 // Get the rd of the vpn instance
 622                 String rd = getRouteDistinguisher(vpnName);
 623
 624                 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 625                 if (nextHopIp == null){
 626                     LOG.error("NextHop for interface {} is null", interfaceName);
 627                     return;
 628                 }
 629
 630                 List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
 631
 632                 LOG.trace("NextHops for interface {} are {}", interfaceName, nextHops);
 633                 for (Adjacency nextHop : nextHops) {
 634                     String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 635                     long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 636                             VpnUtil.getNextHopLabelKey((rd == null) ? vpnName
 637                                     : rd, prefix));
 638                     List<String> adjNextHop = nextHop.getNextHopIpList();
 639                     value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIpList(
 640                             (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : Arrays.asList(nextHopIp))
 641                             .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 642
 643                     if (nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 644                         LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, interfaceName, vpnName);
 645                         writeTxn.merge(
 646                                 LogicalDatastoreType.OPERATIONAL,
 647                                 VpnUtil.getPrefixToInterfaceIdentifier(
 648                                         VpnUtil.getVpnId(broker, vpnName), prefix),
 649                                 VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix), true);
 650                     } else {
 651                         //Extra route adjacency
 652                         LOG.trace("Adding prefix {} and nexthopList {} as extra-route for vpn", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName);
 653                         writeTxn.merge(
 654                                 LogicalDatastoreType.OPERATIONAL,
 655                                 VpnUtil.getVpnToExtrarouteIdentifier(
 656                                         (rd != null) ? rd : vpnName, nextHop.getIpAddress()),
 657                                 VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()), true);
 658                     }
 659                 }
 660
 661                 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 662
 663                 VpnInterface opInterface = VpnUtil.getVpnInterface(interfaceName, vpnName, aug, dpnId, Boolean.FALSE);
 664                 InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 665                 writeTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, true);
 666                 long vpnId = VpnUtil.getVpnId(broker, vpnName);
 667
 668                 for (Adjacency nextHop : aug.getAdjacency()) {
 669                     long label = nextHop.getLabel();
 670                     List<String> nextHopList = new ArrayList<>(nextHop.getNextHopIpList());
 671                     if (rd != null) {
 672
 673                         addToLabelMapper(label, dpnId, nextHop.getIpAddress(), nextHopList, vpnId,
 674                                 interfaceName, null,false, rd);
 675                         addPrefixToBGP(rd, nextHop.getIpAddress(), nextHopIp, label, writeTxn);
 676                         //TODO: ERT - check for VPNs importing my route
 677                         for (VpnInstance vpn : vpnsToImportRoute) {
 678                             String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
 679                             if (vpnRd != null) {
 680                                 LOG.debug("Exporting route with rd {} prefix {} nexthop {} label {} to VPN {}", vpnRd, nextHop.getIpAddress(), nextHopIp, label, vpn);
 681                                 VpnUtil.addFibEntryToDS(broker, vpnRd, nextHop.getIpAddress(), nextHopIp, (int) label, RouteOrigin.SELF_IMPORTED, writeTxn);
 682                             }
 683                         }
 684                     } else {
 685                         // ### add FIB route directly
 686                         VpnUtil.addFibEntryToDS(broker, vpnName, nextHop.getIpAddress(), nextHopIp,
 687                                 (int) label, RouteOrigin.STATIC, writeTxn);
 688                     }
 689                 }
 690             }
 691         }
 692     }
 693
 694     private List<VpnInstance> getVpnsImportingMyRoute(final String vpnName) {
 695         List<VpnInstance> vpnsToImportRoute = new ArrayList<>();
 696
 697         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 698                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 699         Optional<VpnInstance> optVpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 700         final VpnInstance vpnInstance;
 701         if (optVpnInstance.isPresent()) {
 702             vpnInstance = optVpnInstance.get();
 703         } else {
 704             LOG.debug("Could not retrieve vpn instance {} to check for vpns importing the routes", vpnName);
 705             return vpnsToImportRoute;
 706         }
 707
 708         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 709             @Override
 710             public boolean apply(VpnInstance input) {
 711                 return !input.getVpnInstanceName().equals(vpnName);
 712             }
 713         };
 714
 715         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 716             @Override
 717             public boolean apply(VpnInstance input) {
 718                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ExportExtcommunity),
 719                         getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
 720                 return Iterators.size(commonRTs.iterator()) > 0;
 721             }
 722         };
 723
 724         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 725             @Override
 726             public String apply(VpnInstance vpnInstance) {
 727                 //return vpnInstance.getVpnInstanceName();
 728                 return vpnInstance.getIpv4Family().getRouteDistinguisher();
 729             }
 730         };
 731
 732         vpnsToImportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(broker)).
 733                 filter(excludeVpn).
 734                 filter(matchRTs).toList();
 735         return vpnsToImportRoute;
 736     }
 737
 738     private List<VpnInstance> getVpnsExportingMyRoute(final String vpnName) {
 739         List<VpnInstance> vpnsToExportRoute = new ArrayList<>();
 740
 741         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 742                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 743         Optional<VpnInstance> optVpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 744         final VpnInstance vpnInstance;
 745         if (optVpnInstance.isPresent()) {
 746             vpnInstance = optVpnInstance.get();
 747         } else {
 748             LOG.debug("Could not retrieve vpn instance {} to check for vpns exporting the routes", vpnName);
 749             return vpnsToExportRoute;
 750         }
 751
 752         Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
 753             @Override
 754             public boolean apply(VpnInstance input) {
 755                 return !input.getVpnInstanceName().equals(vpnName);
 756             }
 757         };
 758
 759         Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
 760             @Override
 761             public boolean apply(VpnInstance input) {
 762                 Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ImportExtcommunity),
 763                         getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
 764                 return Iterators.size(commonRTs.iterator()) > 0;
 765             }
 766         };
 767
 768         Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
 769             @Override
 770             public String apply(VpnInstance vpnInstance) {
 771                 return vpnInstance.getVpnInstanceName();
 772             }
 773         };
 774
 775         vpnsToExportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(broker)).
 776                 filter(excludeVpn).
 777                 filter(matchRTs).toList();
 778         return vpnsToExportRoute;
 779     }
 780
 781     private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
 782         final Predicate<T> inPredicate = Predicates.<T>in(collection2);
 783         return new Iterable<T>() {
 784             @Override
 785             public Iterator<T> iterator() {
 786                 return Iterators.filter(collection1.iterator(), inPredicate);
 787             }
 788         };
 789     }
 790
 791     private List<String> getRts(VpnInstance vpnInstance, VpnTarget.VrfRTType rtType) {
 792         String name = vpnInstance.getVpnInstanceName();
 793         List<String> rts = new ArrayList<>();
 794         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 795         if (vpnConfig == null) {
 796             LOG.trace("vpn config is not available for {}", name);
 797             return rts;
 798         }
 799         VpnTargets targets = vpnConfig.getVpnTargets();
 800         if (targets == null) {
 801             LOG.trace("vpn targets not available for {}", name);
 802             return rts;
 803         }
 804         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 805         if (vpnTargets == null) {
 806             LOG.trace("vpnTarget values not available for {}", name);
 807             return rts;
 808         }
 809         for (VpnTarget target : vpnTargets) {
 810             //TODO: Check for RT type is Both
 811             if(target.getVrfRTType().equals(rtType) ||
 812                     target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
 813                 String rtValue = target.getVrfRTValue();
 814                 rts.add(rtValue);
 815             }
 816         }
 817         return rts;
 818     }
 819
 820     private List<String> getExportRts(VpnInstance vpnInstance) {
 821         List<String> exportRts = new ArrayList<>();
 822         VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 823         VpnTargets targets = vpnConfig.getVpnTargets();
 824         List<VpnTarget> vpnTargets = targets.getVpnTarget();
 825         for (VpnTarget target : vpnTargets) {
 826             if (target.getVrfRTType().equals(VpnTarget.VrfRTType.ExportExtcommunity)) {
 827                 String rtValue = target.getVrfRTValue();
 828                 exportRts.add(rtValue);
 829             }
 830         }
 831         return exportRts;
 832     }
 833
 834     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 835                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow, WriteTransaction writeTxn){
 836         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 837         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, BigInteger.valueOf(vpnId));
 838         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 839                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 840
 841         // Matching Arp reply flows
 842         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 843         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 844                 metadata, metadataMask }));
 845
 846         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 847
 848         // Instruction to punt to controller
 849         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 850         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 851         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 852         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
 853                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
 854
 855         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 856
 857         // Install the flow entry in L3_INTERFACE_TABLE
 858         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 859                     NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 860         FlowEntity flowEntity;
 861         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 862                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 863                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 864         Flow flow = flowEntity.getFlowBuilder().build();
 865         String flowId = flowEntity.getFlowId();
 866         FlowKey flowKey = new FlowKey( new FlowId(flowId));
 867         Node nodeDpn = buildDpnNode(dpId);
 868
 869         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 870                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 871                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 872
 873         if (writeTxn != null) {
 874             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 875                 LOG.debug("Creating ARP Flow for interface {}", vpnInterfaceName);
 876                 writeTxn.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
 877             } else {
 878                 LOG.debug("Deleting ARP Flow for interface {}", vpnInterfaceName);
 879                 writeTxn.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 880             }
 881         } else {
 882             if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 883                 LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 884                 mdsalManager.installFlow(flowEntity);
 885             } else {
 886                 LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 887                 mdsalManager.removeFlow(flowEntity);
 888             }
 889         }
 890     }
 891
 892     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
 893     private Node buildDpnNode(BigInteger dpnId) {
 894         NodeId nodeId = new NodeId("openflow:" + dpnId);
 895         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
 896
 897         return nodeDpn;
 898     }
 899
 900     private String getRouteDistinguisher(String vpnName) {
 901         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 902                                       .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 903         Optional<VpnInstance> vpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 904         String rd = "";
 905         if(vpnInstance.isPresent()) {
 906             VpnInstance instance = vpnInstance.get();
 907             VpnAfConfig config = instance.getIpv4Family();
 908             rd = config.getRouteDistinguisher();
 909         }
 910         return rd;
 911     }
 912
 913     private void updateMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName,
 914                                   WriteTransaction writeTxn) {
 915         String routeDistinguisher = getRouteDistinguisher(vpnName);
 916         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 917         synchronized (vpnName.intern()) {
 918             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 919             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 920             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op
 921                     .data.entry.vpn.to.dpn.list.VpnInterfaces
 922                     vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 923
 924             if (dpnInVpn.isPresent()) {
 925                 if (writeTxn != null) {
 926                     writeTxn.put(LogicalDatastoreType.OPERATIONAL, id.child(
 927                             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance
 928                                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 929                             new VpnInterfacesKey(intfName)), vpnInterface, true);
 930                 } else {
 931                     VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 932                             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance
 933                                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 934                             new VpnInterfacesKey(intfName)), vpnInterface);
 935                 }
 936             } else {
 937
 938                 /*
 939                 * Increment the active dpn count in VpnInstanceOpData.
 940                 * ActiveDpnCount will be decremented when all the VpnInt in that Dpn are removed.
 941                 */
 942                 InstanceIdentifier<VpnInstanceOpDataEntry> vpnInsOpDataId = VpnUtil.getVpnInstanceOpDataIdentifier(rd);
 943                 Optional<VpnInstanceOpDataEntry> vpnInstOpData = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, vpnInsOpDataId);
 944
 945                 java.lang.Long activeDpnCnt = vpnInstOpData.get().getActiveDpnCount();
 946
 947                 VpnInstanceOpDataEntryBuilder builder =
 948                         new VpnInstanceOpDataEntryBuilder(vpnInstOpData.get()).setVrfId(rd).setVpnId(vpnId).setVpnInstanceName(vpnName).setActiveDpnCount(++activeDpnCnt);
 949
 950                 if (writeTxn != null) {
 951                     writeTxn.merge(LogicalDatastoreType.OPERATIONAL,
 952                             vpnInsOpDataId,
 953                             builder.build(), true);
 954                 } else {
 955                     VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 956                             vpnInsOpDataId,
 957                             builder.build());
 958                 }
 959
 960                 VpnToDpnListBuilder vpnToDpnList = new VpnToDpnListBuilder().setDpnId(dpnId).setDpnState(VpnToDpnList.DpnState.Active);
 961                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
 962                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = new ArrayList<>();
 963                 vpnInterfaces.add(vpnInterface);
 964                 if (writeTxn != null) {
 965                     writeTxn.merge(LogicalDatastoreType.OPERATIONAL, id,
 966                             vpnToDpnList.setVpnInterfaces(vpnInterfaces).build(), true);
 967                 } else {
 968                     VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, id,
 969                             vpnToDpnList.setVpnInterfaces(vpnInterfaces).build());
 970                 }
 971                 /**
 972                  * FIXME: DC Gateway tunnel should be built dynamically
 973                  //this is the first VM in this VPN on the DPN, may be a new DPN has come up,
 974                  //if tunnel to DC GW does not exist, create it
 975                  //if(!tunnelExists(dpnID, bgpManager.getDCGWIP()))
 976                  String dcGW = bgpManager.getDCGwIP();
 977                  if(dcGW != null && !dcGW.isEmpty())
 978                  {
 979                  LOG.debug("Building tunnel from DPN {} to DC GW {}", dpnId, dcGW);
 980                  itmProvider.buildTunnelFromDPNToDCGW(dpnId, new IpAddress(dcGW.toCharArray()));
 981                  }*/
 982                 fibManager.populateFibOnNewDpn(dpnId, vpnId, (rd == null) ? vpnName : rd);
 983                 publishAddNotification(dpnId, vpnName, rd);
 984                 //TODO: IRT - import local routes to vpn. check for the VPNs exporting the routes
 985                 //FIXME: do we need to handle here. since already routes are imported to this vpn
 986
 987             }
 988         }
 989     }
 990
 991     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
 992         List<VpnInstance> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
 993         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 994         for (VpnInstance vpn : vpnsToExportRoute) {
 995             String rd = vpn.getIpv4Family().getRouteDistinguisher();
 996             long exportingVpnId = VpnUtil.getVpnId(broker, vpn.getVpnInstanceName());
 997             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(broker, vpn.getIpv4Family().getRouteDistinguisher());
 998             if (vrfEntries != null) {
 999                 for (VrfEntry vrfEntry : vrfEntries) {
1000                     try {
1001                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1002                             continue;
1003                         }
1004                         String prefix = vrfEntry.getDestPrefix();
1005                         long label = vrfEntry.getLabel();
1006                         List<String> nextHops = vrfEntry.getNextHopAddressList();
1007                         SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1008                         for (String nh : nextHops) {
1009                             if (route != null) {
1010                                 LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1011                                 importSubnetRouteForNewVpn(rd, prefix, nh, (int)label, route);
1012                             } else {
1013                                 LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
1014                                 VpnUtil.addFibEntryToDS(broker, vpnRd, prefix, nh, (int)label, RouteOrigin
1015                                         .SELF_IMPORTED, null);
1016                             }
1017                         }
1018                     } catch (Exception e) {
1019                         LOG.error("Exception occurred while importing route with prefix {} label {} nexthop {} from vpn {} to vpn {}", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpn.getVpnInstanceName(), vpnName);
1020                     }
1021                 }
1022             } else {
1023                 LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getIpv4Family().getRouteDistinguisher());
1024             }
1025         }
1026     }
1027
1028     private void removeFromMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName,
1029                                       WriteTransaction writeTxn) {
1030         //TODO: Delay 'DPN' removal so that other services can cleanup the entries for this dpn
1031         synchronized (vpnName.intern()) {
1032             String rd = VpnUtil.getVpnRd(broker, vpnName);
1033             InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
1034             Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
1035             if (dpnInVpn.isPresent()) {
1036                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1037                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1038                 org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn
1039                         .instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
1040                         currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1041
1042                 if (vpnInterfaces.remove(currVpnInterface)) {
1043                     if (vpnInterfaces.isEmpty()) {
1044                         List<IpAddresses> ipAddresses = dpnInVpn.get().getIpAddresses();
1045                         if (ipAddresses == null || ipAddresses.isEmpty()) {
1046
1047                            /*
1048                            * Mark the DPN state as Inactive as the Vpnintf list is empty.
1049                            * This DPN will be removed in DpnToVpnChangeListner only if its
1050                            * inactive and ActiveDpnCount is zero.
1051                            */
1052                             VpnToDpnListBuilder vpnToDpnList =
1053                                     new VpnToDpnListBuilder(dpnInVpn.get())
1054                                             .setDpnId(dpnId)
1055                                             .setDpnState(VpnToDpnList.DpnState.Inactive)
1056                                             .setVpnInterfaces(vpnInterfaces);
1057
1058                             if (writeTxn != null) {
1059                                 writeTxn.merge(LogicalDatastoreType.OPERATIONAL, id , vpnToDpnList.build(), true);
1060                             } else {
1061                                 VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, id, vpnToDpnList.build());
1062                             }
1063
1064                             InstanceIdentifier<VpnInstanceOpDataEntry> vpnInstOpDataId = VpnUtil.getVpnInstanceOpDataIdentifier(rd);
1065                             Optional<VpnInstanceOpDataEntry> vpnInstOpData = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, vpnInstOpDataId);
1066
1067                             java.lang.Long activeDpnCnt = vpnInstOpData.get().getActiveDpnCount();
1068
1069                             VpnInstanceOpDataEntryBuilder builder =
1070                                     new VpnInstanceOpDataEntryBuilder(vpnInstOpData.get()).setVrfId(rd).setVpnId(vpnId).setVpnInstanceName(vpnName).setActiveDpnCount(--activeDpnCnt);
1071
1072                             if (writeTxn != null) {
1073                                 writeTxn.merge(LogicalDatastoreType.OPERATIONAL, vpnInstOpDataId , builder.build(), true);
1074                             } else {
1075                                 VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, vpnInstOpDataId, builder.build());
1076                             }
1077
1078                             LOG.debug("Sending cleanup event for dpn {} in VPN {}", dpnId, vpnName);
1079                             fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
1080                             publishRemoveNotification(dpnId, vpnName, rd);
1081                         } else {
1082                             LOG.debug("vpn interfaces are empty but ip addresses are present for the vpn {} in dpn {}", vpnName, dpnId);
1083                         }
1084                     } else {
1085                         if (writeTxn != null) {
1086                             writeTxn.delete(LogicalDatastoreType.OPERATIONAL, id.child(
1087                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op
1088                                             .data
1089                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1090                                     new VpnInterfacesKey(intfName)));
1091                         } else {
1092                             VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
1093                                     org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
1094                                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
1095                                     new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
1096                         }
1097                     }
1098                 }
1099             }
1100         }
1101     }
1102
1103     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label, WriteTransaction writeTxn) {
1104
1105         try {
1106             LOG.info("ADD: Adding Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1107             VpnUtil.addFibEntryToDS(broker, rd, prefix, nextHopIp, (int)label, RouteOrigin.STATIC, writeTxn);
1108             bgpManager.advertisePrefix(rd, prefix, Arrays.asList(nextHopIp), (int)label);
1109             LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
1110         } catch(Exception e) {
1111             LOG.error("Add prefix failed", e);
1112         }
1113     }
1114
1115
1116     private InstanceIdentifier<VpnInterface> getWildCardPath() {
1117         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
1118     }
1119
1120     @Override
1121     public void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1122         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
1123         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1124         final String interfaceName = key.getName();
1125
1126         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1127         Optional<VpnInterface> existingVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1128         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
1129             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
1130         if (existingVpnInterface.isPresent()){
1131             BigInteger dpnId = BigInteger.ZERO;
1132             Boolean dpnIdRetrieved = Boolean.FALSE;
1133             if(interfaceState != null){
1134                 try{
1135                     dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1136                     dpnIdRetrieved = Boolean.TRUE;
1137                 }catch (Exception e){
1138                     LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", interfaceName, e);
1139                 }
1140             } else {
1141                 LOG.error("Unable to retrieve interfaceState for interface {} , quitting ", interfaceName);
1142                 return;
1143             }
1144             if(dpnIdRetrieved == Boolean.FALSE){
1145                 LOG.info("dpnId for {} has not been retrieved yet. Fetching from vpn interface operational DS", interfaceName);
1146                 dpnId = existingVpnInterface.get().getDpnId();
1147             }
1148             final int ifIndex = interfaceState.getIfIndex();
1149             final BigInteger dpId = dpnId;
1150             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1151             dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1152                     new Callable<List<ListenableFuture<Void>>>() {
1153                         @Override
1154                         public List<ListenableFuture<Void>> call() throws Exception {
1155                             WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
1156                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, false, true, writeTxn);
1157                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1158                             futures.add(writeTxn.submit());
1159                             waitForFibToRemoveVpnPrefix(interfaceName);
1160                             return futures;
1161                         }
1162                     });
1163             //processVpnInterfaceDown(dpnId, interfaceName, interfaceState.getIfIndex(), false, true);
1164         }else{
1165             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
1166         }
1167     }
1168
1169     protected void processVpnInterfaceDown(BigInteger dpId, String interfaceName, int lPortTag, boolean isInterfaceStateDown,
1170                                            boolean isConfigRemoval, WriteTransaction writeTxn) {
1171         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1172         if (!isInterfaceStateDown) {
1173             synchronized (interfaceName.intern()) {
1174                 VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
1175                 if(vpnInterface == null){
1176                     LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
1177                     return;
1178                 }else{
1179                     final String vpnName = vpnInterface.getVpnInstanceName();
1180                     if(!vpnInterface.isScheduledForRemove()){
1181                         VpnUtil.updateVpnInterface(broker, interfaceName, dpId, vpnName, Boolean.TRUE, writeTxn);
1182                         removeAdjacenciesFromVpn(dpId, interfaceName, vpnInterface.getVpnInstanceName(), writeTxn);
1183                         LOG.info("Unbinding vpn service from interface {} ", interfaceName);
1184                         unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown, isConfigRemoval, writeTxn);
1185                     }else{
1186                         LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ", interfaceName);
1187                         return;
1188                     }
1189                 }
1190             }
1191         } else {
1192                 synchronized (interfaceName.intern()) {
1193                     // Interface is retained in the DPN, but its Link Down.
1194                     // Only withdraw the prefixes for this interface from BGP
1195                     VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
1196                     if(vpnInterface == null){
1197                         LOG.info("Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in operational data store", interfaceName);
1198                         return;
1199                     }else {
1200                         withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
1201                     }
1202                 }
1203         }
1204
1205     }
1206
1207     private void waitForFibToRemoveVpnPrefix(String interfaceName) {
1208         // FIB didn't get a chance yet to clean up this VPNInterface
1209         // Let us give it a chance here !
1210         LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
1211         try {
1212             Runnable notifyTask = new VpnNotifyTask();
1213             vpnIntfMap.put(interfaceName, notifyTask);
1214             synchronized (notifyTask) {
1215                 try {
1216                     notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
1217                 } catch (InterruptedException e) {
1218                 }
1219             }
1220         } finally {
1221             vpnIntfMap.remove(interfaceName);
1222         }
1223     }
1224
1225     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final String interfaceName, final String vpnName,
1226                                           WriteTransaction writeTxn) {
1227         //Read NextHops
1228         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
1229         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1230         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
1231
1232         String rd = VpnUtil.getVpnRd(broker, vpnName);
1233         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", interfaceName,
1234                 vpnName, rd);
1235         if (adjacencies.isPresent()) {
1236             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1237
1238             if (!nextHops.isEmpty()) {
1239                 LOG.trace("NextHops are " + nextHops);
1240                 for (Adjacency nextHop : nextHops) {
1241                     List<String> nhList = new ArrayList<String>();
1242                     if (nextHop.getMacAddress() == null || nextHop.getMacAddress().isEmpty()) {
1243                         // This is either an extra-route (or) a learned IP via subnet-route
1244                         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
1245                         if (nextHopIp == null || nextHopIp.isEmpty()) {
1246                             LOG.error("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
1247                                     rd, nextHop.getIpAddress());
1248                             continue;
1249                         }
1250                         nhList = Arrays.asList(nextHopIp);
1251                     } else {
1252                         // This is a primary adjacency
1253                         nhList = nextHop.getNextHopIpList();
1254                     }
1255                     if (rd.equals(vpnName)) {
1256                         //this is an internal vpn - the rd is assigned to the vpn instance name;
1257                         //remove from FIB directly
1258                         for(String nh : nhList) {
1259                             VpnUtil.removeFibEntryFromDS(broker, vpnName, nextHop.getIpAddress(), nh, writeTxn);
1260                         }
1261                     } else {
1262                         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1263                         for (String nh : nextHop.getNextHopIpList()) {
1264                             //IRT: remove routes from other vpns importing it
1265                             removePrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeTxn);
1266                             for (VpnInstance vpn : vpnsToImportRoute) {
1267                                 String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
1268                                 if (vpnRd != null) {
1269                                     LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd, nextHop.getIpAddress(), vpn.getVpnInstanceName());
1270                                     VpnUtil.removeFibEntryFromDS(broker, vpnRd, nextHop.getIpAddress(), nh, writeTxn);
1271                                 }
1272                             }
1273                         }
1274                     }
1275                     String ip = nextHop.getIpAddress().split("/")[0];
1276                     VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(broker,
1277                             vpnName, ip);
1278                     if (vpnPortipToPort != null && !vpnPortipToPort.isConfig()) {
1279                         LOG.trace("VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
1280                                 vpnPortipToPort.getPortName(),ip);
1281                         VpnUtil.removeVpnPortFixedIpToPort(broker, vpnName, ip);
1282                     }
1283                 }
1284             }
1285         }
1286     }
1287
1288
1289     private void unbindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName,
1290                                int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval,
1291                                WriteTransaction writeTxn) {
1292         if (!isInterfaceStateDown && isConfigRemoval) {
1293             writeTxn.delete(LogicalDatastoreType.CONFIGURATION,
1294                     InterfaceUtils.buildServiceId(vpnInterfaceName,
1295                             VpnConstants.L3VPN_SERVICE_IDENTIFIER));
1296         }
1297         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
1298         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
1299                 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW, writeTxn);
1300         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
1301                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.DEL_FLOW, writeTxn);
1302     }
1303
1304
1305     private void removePrefixFromBGP(String rd, String prefix, String nextHop, WriteTransaction writeTxn) {
1306         VpnUtil.removeFibEntryFromDS(broker, rd, prefix, nextHop, writeTxn);
1307         try {
1308             LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
1309             VpnUtil.removeFibEntryFromDS(broker, rd, prefix, nextHop, writeTxn);
1310             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1311             LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
1312         } catch(Exception e) {
1313             LOG.error("Delete prefix failed", e);
1314         }
1315     }
1316
1317     @Override
1318     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
1319         LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
1320         String oldVpnName = original.getVpnInstanceName();
1321         String newVpnName = update.getVpnInstanceName();
1322         BigInteger dpnId = update.getDpnId();
1323         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
1324         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
1325         if (oldAdjs == null) {
1326             oldAdjs = new ArrayList<>();
1327         }
1328         if (newAdjs == null) {
1329             newAdjs = new ArrayList<>();
1330         }
1331         //handles switching between <internal VPN - external VPN>
1332         if (!oldVpnName.equals(newVpnName)) {
1333             remove(identifier, original);
1334             add(identifier, update);
1335         }
1336         //handle both addition and removal of adjacencies
1337         //currently, new adjacency may be an extra route
1338         if (!oldAdjs.equals(newAdjs)) {
1339             for (Adjacency adj : newAdjs) {
1340                 if (oldAdjs.contains(adj)) {
1341                     oldAdjs.remove(adj);
1342                 } else {
1343                     // add new adjacency - right now only extra route will hit this path
1344                     addNewAdjToVpnInterface(identifier, adj, dpnId);
1345                 }
1346             }
1347             for (Adjacency adj : oldAdjs) {
1348                 delAdjFromVpnInterface(identifier, adj, dpnId);
1349             }
1350         }
1351     }
1352
1353     public void processArpRequest(IpAddress srcIP, PhysAddress srcMac, IpAddress targetIP, PhysAddress targetMac,String srcInterface){
1354         //Build ARP response with ARP requests TargetIp TargetMac as the Arp Response SrcIp and SrcMac
1355         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
1356                 .setDstIpaddress(srcIP).setDstMacaddress(srcMac).setSrcIpaddress(targetIP).setSrcMacaddress(targetMac).build();
1357         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
1358         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
1359         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
1360             @Override
1361             public void onFailure(Throwable error) {
1362                 LOG.error("Error - {}", msgFormat, error);
1363             }
1364
1365             @Override
1366             public void onSuccess(RpcResult<Void> result) {
1367                 if(!result.isSuccessful()) {
1368                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
1369                 } else {
1370                     LOG.debug("Successful RPC Result - {}", msgFormat);
1371                 }
1372             }
1373         });
1374     }
1375
1376     private String getErrorText(Collection<RpcError> errors) {
1377         StringBuilder errorText = new StringBuilder();
1378         for(RpcError error : errors) {
1379             errorText.append(",").append(error.getErrorType()).append("-")
1380                      .append(error.getMessage());
1381         }
1382         return errorText.toString();
1383     }
1384
1385     private void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
1386                                   String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd) {
1387         Preconditions.checkNotNull(label, "label cannot be null or empty!");
1388         Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
1389         Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
1390         Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
1391         if (!isSubnetRoute) {
1392             // NextHop must be present for non-subnetroute entries
1393             Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
1394         }
1395         LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
1396         if (dpnId != null) {
1397             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1398                     .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
1399             LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
1400             lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(vpnId)
1401                     .setIsSubnetRoute(isSubnetRoute);
1402             if (elanTag != null) {
1403                 lriBuilder.setElanTag(elanTag);
1404             }
1405             if (vpnInterfaceName != null) {
1406                 lriBuilder.setVpnInterfaceName(vpnInterfaceName);
1407             }
1408             lriBuilder.setParentVpnRd(rd);
1409             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(broker, rd);
1410             if (vpnInstanceOpDataEntry != null) {
1411                 List<String> vpnInstanceNames = Arrays.asList(vpnInstanceOpDataEntry.getVpnInstanceName());
1412                 lriBuilder.setVpnInstanceList(vpnInstanceNames);
1413             }
1414             LabelRouteInfo lri = lriBuilder.build();
1415             LOG.trace("Adding route info to label map: {}", lri);
1416             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
1417         } else {
1418             LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
1419         }
1420     }
1421
1422     public synchronized void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
1423                                                         long elantag, BigInteger dpnId, WriteTransaction writeTxn) {
1424
1425         SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
1426         RouteOrigin origin = RouteOrigin.STATIC; // Only case when a route is considered as directly connected
1427         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1428                                                  .setLabel((long)label).setOrigin(origin.getValue())
1429                                                  .addAugmentation(SubnetRoute.class, route).build();
1430
1431         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
1432
1433         //TODO: What should be parentVpnId? Get it from RD?
1434         long vpnId = VpnUtil.getVpnId(broker, vpnName);
1435         addToLabelMapper((long)label, dpnId, prefix, Arrays.asList(nextHop), vpnId, null, elantag, true, rd);
1436         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1437
1438         InstanceIdentifierBuilder<VrfTables> idBuilder =
1439                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1440         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1441
1442         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1443                 setVrfEntry(vrfEntryList).build();
1444
1445         if (writeTxn != null) {
1446             writeTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1447         } else {
1448             VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1449         }
1450
1451         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1452         if (vpnsToImportRoute.size() > 0) {
1453             VrfEntry importingVrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1454                     .setLabel((long) label).setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
1455                     .addAugmentation(SubnetRoute.class, route).build();
1456             List<VrfEntry> importingVrfEntryList = Arrays.asList(importingVrfEntry);
1457             for (VpnInstance vpnInstance : vpnsToImportRoute) {
1458                 LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop, label, vpnInstance.getVpnInstanceName());
1459                 String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1460                 InstanceIdentifier<VrfTables> importingVrfTableId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(importingRd)).build();
1461                 VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(importingVrfEntryList).build();
1462                 if (writeTxn != null) {
1463                     writeTxn.merge(LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable, true);
1464                 } else {
1465                     VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable);
1466                 }
1467             }
1468         }
1469     }
1470
1471     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1472                                                         SubnetRoute route) {
1473
1474         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1475         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
1476                 .setLabel((long)label).setOrigin(origin.getValue())
1477                 .addAugmentation(SubnetRoute.class, route).build();
1478         LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, route.getElantag());
1479         List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
1480         InstanceIdentifierBuilder<VrfTables> idBuilder =
1481                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1482         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1483         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
1484                 setVrfEntry(vrfEntryList).build();
1485         VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1486     }
1487
1488     public synchronized void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName){
1489         VpnUtil.removeFibEntryFromDS(broker, rd, prefix, null /* nextHopToRemove */, null);
1490         List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
1491         for (VpnInstance vpnInstance : vpnsToImportRoute) {
1492             String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
1493             LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix, vpnInstance.getVpnInstanceName());
1494             VpnUtil.removeFibEntryFromDS(broker, importingRd, prefix, null, null);
1495         }
1496     }
1497
1498     public synchronized void removeVrfFromDS(String rd) {
1499         LOG.debug("Removing vrf table for rd {}", rd);
1500
1501         InstanceIdentifierBuilder<VrfTables> idBuilder =
1502                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1503         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1504
1505         VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, VpnUtil.DEFAULT_CALLBACK);
1506
1507     }
1508
1509     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1510
1511         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
1512
1513         if (optVpnInterface.isPresent()) {
1514             VpnInterface currVpnIntf = optVpnInterface.get();
1515             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1516             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1517             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
1518             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, adjPath);
1519             long label =
1520                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1521                             VpnUtil.getNextHopLabelKey((rd != null) ? rd : currVpnIntf.getVpnInstanceName(), prefix));
1522
1523             List<Adjacency> adjacencies;
1524             if (optAdjacencies.isPresent()) {
1525                 adjacencies = optAdjacencies.get().getAdjacency();
1526             } else {
1527                 //This code will not be hit since VM adjacency will always be there
1528                 adjacencies = new ArrayList<>();
1529             }
1530
1531
1532             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIpList(adj.getNextHopIpList())
1533                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
1534             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1535             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove());
1536             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1537             for (String nh : adj.getNextHopIpList()) {
1538                 addExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(), (int) label,
1539                               currVpnIntf.getName());
1540             }
1541
1542         }
1543
1544     }
1545
1546     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
1547         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
1548
1549         if (optVpnInterface.isPresent()) {
1550             VpnInterface currVpnIntf = optVpnInterface.get();
1551
1552             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
1553             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
1554             if (optAdjacencies.isPresent()) {
1555                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1556
1557                 if (!adjacencies.isEmpty()) {
1558                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
1559                     LOG.trace("Adjacencies are " + adjacencies);
1560                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1561                     while (adjIt.hasNext()) {
1562                         Adjacency adjElem = adjIt.next();
1563                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1564                             adjIt.remove();
1565
1566                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
1567                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
1568                                                                               currVpnIntf.getVpnInstanceName(),
1569                                                                               aug, dpnId, currVpnIntf.isScheduledForRemove());
1570
1571                             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
1572
1573                             for (String nh : adj.getNextHopIpList()) {
1574                                 delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
1575                                               currVpnIntf.getName());
1576                             }
1577                             break;
1578                         }
1579
1580                     }
1581                 }
1582             }
1583         }
1584
1585     }
1586
1587     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label,
1588                                  String intfName) {
1589
1590         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
1591         VpnUtil.syncUpdate(
1592             broker,
1593             LogicalDatastoreType.OPERATIONAL,
1594             VpnUtil.getVpnToExtrarouteIdentifier( (rd != null) ? rd : routerID, destination),
1595             VpnUtil.getVpnToExtraroute(destination, Arrays.asList(nextHop)));
1596
1597         BigInteger dpnId = null;
1598         if (intfName != null && !intfName.isEmpty()) {
1599             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1600             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
1601             if (nextHopIp == null || nextHopIp.isEmpty()) {
1602                 LOG.error("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1603                           intfName, destination);
1604                 return;
1605             }
1606             nextHop = nextHopIp;
1607         }
1608         List<String> nextHopIpList = Arrays.asList(nextHop);
1609         addToLabelMapper((long)label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(broker, routerID),
1610                 intfName, null, false, rd);
1611
1612         // TODO (eperefr): This is a limitation to be stated in docs. When configuring static route to go to
1613         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
1614         // first place.
1615         InterVpnLink interVpnLink = VpnUtil.getInterVpnLinkByEndpointIp(broker, nextHop);
1616         if ( interVpnLink != null ) {
1617             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
1618             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
1619             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
1620             // This is like leaking one of the Vpn2 routes towards Vpn1
1621             boolean nexthopIsVpn2 = ( interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nextHop) );
1622             String srcVpnUuid = (nexthopIsVpn2) ? interVpnLink.getSecondEndpoint().getVpnUuid().getValue()
1623                                                 : interVpnLink.getFirstEndpoint().getVpnUuid().getValue();
1624             String dstVpnUuid = (nexthopIsVpn2) ? interVpnLink.getFirstEndpoint().getVpnUuid().getValue()
1625                                                 : interVpnLink.getSecondEndpoint().getVpnUuid().getValue();
1626             String dstVpnRd = VpnUtil.getVpnRd(broker, dstVpnUuid);
1627             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
1628                                                 VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
1629             VpnUtil.leakRoute(broker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel);
1630         } else {
1631             if (rd != null) {
1632                 addPrefixToBGP(rd, destination, nextHop, label, null);
1633             } else {
1634                 // ### add FIB route directly
1635                 VpnUtil.addFibEntryToDS(broker, routerID, destination, nextHop, label, RouteOrigin.STATIC, null);
1636             }
1637         }
1638     }
1639
1640     protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName) {
1641         if (intfName != null && !intfName.isEmpty()) {
1642             BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
1643             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
1644             if (nextHopIp == null || nextHopIp.isEmpty()) {
1645                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
1646                         intfName, destination);
1647             }
1648             nextHop = nextHopIp;
1649         }
1650
1651         if (rd != null) {
1652             removePrefixFromBGP(rd, destination, nextHop, null);
1653         } else {
1654             // ### add FIB route directly
1655             VpnUtil.removeFibEntryFromDS(broker, routerID, destination, nextHop, null);
1656         }
1657     }
1658
1659     class VpnInterfaceOpListener extends AbstractDataChangeListener<VpnInterface> {
1660
1661         public VpnInterfaceOpListener() {
1662             super(VpnInterface.class);
1663         }
1664
1665         @Override
1666         protected void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface del) {
1667             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1668             String interfaceName = key.getName();
1669             String vpnName = del.getVpnInstanceName();
1670
1671             LOG.trace("VpnInterfaceOpListener removed: interface name {} vpnName {}", interfaceName, vpnName);
1672             //decrement the vpn interface count in Vpn Instance Op Data
1673             InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance>
1674                     id = VpnUtil.getVpnInstanceToVpnIdIdentifier(vpnName);
1675             Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance> vpnInstance
1676                     = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
1677
1678             if (vpnInstance.isPresent()) {
1679                 String rd = null;
1680                 rd = vpnInstance.get().getVrfId();
1681                 //String rd = getRouteDistinguisher(del.getVpnInstanceName());
1682
1683                 VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
1684                 LOG.trace("VpnInterfaceOpListener removed: interface name {} rd {} vpnName {} in Vpn Op Instance {}",
1685                         interfaceName, rd, vpnName, vpnInstOp);
1686
1687                 if (vpnInstOp != null) {
1688                     // Vpn Interface removed => No more adjacencies from it.
1689                     // Hence clean up interface from vpn-dpn-interface list.
1690                     Adjacency adjacency = del.getAugmentation(Adjacencies.class).getAdjacency().get(0);
1691                     List<Prefixes> prefixToInterface = new ArrayList<>();
1692                     Optional<Prefixes> prefix = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1693                             VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1694                                     VpnUtil.getIpPrefix(adjacency.getIpAddress())));
1695                     if (prefix.isPresent()) {
1696                         prefixToInterface.add(prefix.get());
1697                     }
1698                     if (prefixToInterface.isEmpty()) {
1699                         for (String nh : adjacency.getNextHopIpList()) {
1700                             prefix = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1701                                      VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1702                                                                            VpnUtil.getIpPrefix(nh)));
1703                             if (prefix.isPresent())
1704                                 prefixToInterface.add(prefix.get());
1705                         }
1706                     }
1707                     for (Prefixes pref : prefixToInterface) {
1708                         VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
1709                                        VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(), pref.getIpAddress()),
1710                                        VpnUtil.DEFAULT_CALLBACK);
1711                         synchronized (interfaceName.intern()) {
1712                             updateDpnDbs(pref.getDpnId(), del.getVpnInstanceName(), interfaceName, false, null);
1713                         }
1714                     }
1715                 }
1716             } else {
1717                 LOG.error("rd not retrievable as vpninstancetovpnid for vpn {} is absent, trying rd as ", vpnName, vpnName);
1718             }
1719             notifyTaskIfRequired(interfaceName);
1720         }
1721
1722         private void notifyTaskIfRequired(String intfName) {
1723             Runnable notifyTask = vpnIntfMap.remove(intfName);
1724             if (notifyTask == null) {
1725                 LOG.trace("VpnInterfaceOpListener update: No Notify Task queued for vpnInterface {}", intfName);
1726                 return;
1727             }
1728             executorService.execute(notifyTask);
1729         }
1730
1731         @Override
1732         protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
1733             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1734             String interfaceName = key.getName();
1735
1736             if (original.getVpnInstanceName().equals(update.getVpnInstanceName())) {
1737                 return;
1738             }
1739
1740             //increment the vpn interface count in Vpn Instance Op Data
1741             VpnInstanceOpDataEntry vpnInstOp = null;
1742             InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance>
1743                     origId = VpnUtil.getVpnInstanceToVpnIdIdentifier(original.getVpnInstanceName());
1744             Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance> origVpnInstance
1745                     = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, origId);
1746
1747             if (origVpnInstance.isPresent()) {
1748                 String rd = null;
1749                 rd = origVpnInstance.get().getVrfId();
1750
1751                 vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
1752                 LOG.trace("VpnInterfaceOpListener updated: interface name {} original rd {} original vpnName {}",
1753                         interfaceName, rd, original.getVpnInstanceName());
1754
1755                 if (vpnInstOp != null) {
1756                     Adjacency adjacency = original.getAugmentation(Adjacencies.class).getAdjacency().get(0);
1757                     List<Prefixes> prefixToInterfaceList = new ArrayList<>();
1758                     Optional<Prefixes> prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1759                             VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1760                                     VpnUtil.getIpPrefix(adjacency.getIpAddress())));
1761                     if (prefixToInterface.isPresent()) {
1762                         prefixToInterfaceList.add(prefixToInterface.get());
1763                     } else {
1764                         for (String adj : adjacency.getNextHopIpList()) {
1765                             prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1766                                           VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
1767                                           VpnUtil.getIpPrefix(adj)));
1768                             if (prefixToInterface.isPresent()) {
1769                                 prefixToInterfaceList.add(prefixToInterface.get());
1770                             }
1771                         }
1772                     }
1773                     synchronized (interfaceName.intern()) {
1774                         for (Prefixes prefix : prefixToInterfaceList) {
1775                             updateDpnDbs(prefix.getDpnId(), original.getVpnInstanceName(), interfaceName, false, null);
1776                         }
1777                     }
1778                 }
1779             }
1780             notifyTaskIfRequired(interfaceName);
1781         }
1782
1783         @Override
1784         protected void add(InstanceIdentifier<VpnInterface> identifier, VpnInterface add) {
1785
1786         }
1787     }
1788
1789     protected void handlePrefixesForDPNs(StateTunnelList stateTunnelList, UpdateRouteAction action) {
1790         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1791         BigInteger destDpnId;
1792         String srcTepIp =  String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1793         String destTepIp = String.valueOf(stateTunnelList.getDstInfo().getTepIp().getValue());
1794
1795         InstanceIdentifierBuilder<VpnInstances> idBuilder = InstanceIdentifier.builder(VpnInstances.class);
1796         InstanceIdentifier<VpnInstances> vpnInstancesId = idBuilder.build();
1797         Optional<VpnInstances> vpnInstances = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnInstancesId);
1798         long tunTypeVal = 0, vpnId;
1799
1800         if (stateTunnelList.getDstInfo().getTepDeviceType() == TepTypeInternal.class) {
1801             tunTypeVal = VpnConstants.ITMTunnelLocType.Internal.getValue();
1802         } else if (stateTunnelList.getDstInfo().getTepDeviceType() == TepTypeExternal.class) {
1803             tunTypeVal = VpnConstants.ITMTunnelLocType.External.getValue();
1804         } else if (stateTunnelList.getDstInfo().getTepDeviceType() == TepTypeHwvtep.class){
1805             tunTypeVal = VpnConstants.ITMTunnelLocType.Hwvtep.getValue();
1806         } else {
1807             tunTypeVal = VpnConstants.ITMTunnelLocType.Invalid.getValue();
1808         }
1809         LOG.trace("tunTypeVal is {}", tunTypeVal);
1810         long dcgwPresentStatus = VpnConstants.DCGWPresentStatus.Invalid.getValue();
1811         if (tunTypeVal == VpnConstants.ITMTunnelLocType.External.getValue()) {
1812             Future<RpcResult<IsDcgwPresentOutput>> result;
1813             try {
1814                 result = itmProvider.isDcgwPresent(new IsDcgwPresentInputBuilder()
1815                         .setDcgwIp(destTepIp)
1816                         .build());
1817                 RpcResult<IsDcgwPresentOutput> rpcResult = result.get();
1818                 if (!rpcResult.isSuccessful()) {
1819                     LOG.warn("RPC Call to isDcgwPresent {} returned with Errors {}", destTepIp, rpcResult.getErrors());
1820                 } else {
1821                     dcgwPresentStatus = rpcResult.getResult().getRetVal();
1822                 }
1823             } catch (Exception e) {
1824                 LOG.warn("Exception {} when querying for isDcgwPresent {}, trace {}", e, destTepIp, e.getStackTrace());
1825             }
1826         }
1827
1828         if (vpnInstances.isPresent()) {
1829             List<VpnInstance> vpnInstanceList = vpnInstances.get().getVpnInstance();
1830             Iterator<VpnInstance> vpnInstIter = vpnInstanceList.iterator();
1831             LOG.trace("vpnInstIter {}", vpnInstIter);
1832             while (vpnInstIter.hasNext()) {
1833                 VpnInstance vpnInstance = vpnInstIter.next();
1834                 LOG.trace("vpnInstance {}", vpnInstance);
1835                 vpnId = VpnUtil.getVpnId(broker, vpnInstance.getVpnInstanceName());
1836                 try {
1837                     VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
1838                     LOG.trace("vpnConfig {}", vpnConfig);
1839                     String rd = vpnConfig.getRouteDistinguisher();
1840                     if (rd == null || rd.isEmpty()) {
1841                         rd = vpnInstance.getVpnInstanceName();
1842                         LOG.trace("rd is null or empty. Assigning VpnInstanceName to rd {}", rd);
1843                     }
1844                     InstanceIdentifier<VpnToDpnList> srcId =
1845                         VpnUtil.getVpnToDpnListIdentifier(rd, srcDpnId);
1846                     Optional<VpnToDpnList> srcDpnInVpn =
1847                             VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, srcId);
1848                     if (tunTypeVal == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
1849                         destDpnId = new BigInteger(stateTunnelList.getDstInfo().getTepDeviceId());
1850                         InstanceIdentifier<VpnToDpnList> destId =
1851                                 VpnUtil.getVpnToDpnListIdentifier(rd, destDpnId);
1852                         Optional<VpnToDpnList> destDpnInVpn =
1853                                 VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, destId);
1854                         if (!(srcDpnInVpn.isPresent() &&
1855                                 destDpnInVpn.isPresent())) {
1856                             LOG.trace(" srcDpn {} - destDPN {}, do not share the VPN {} with rd {}.",
1857                                     srcDpnId, destDpnId, vpnInstance.getVpnInstanceName(), rd);
1858                             continue;
1859                         }
1860                     }
1861                     if (srcDpnInVpn.isPresent()) {
1862                         List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
1863                                 .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces>
1864                             vpnInterfaces = srcDpnInVpn.get().getVpnInterfaces();
1865                         for (org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
1866                                 .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces vpnInterface : vpnInterfaces) {
1867                             InstanceIdentifier<VpnInterface> vpnIntfId =
1868                                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getInterfaceName());
1869                             LOG.trace("vpnInterface {}", vpnInterface);
1870                             InstanceIdentifier<Adjacencies> path =
1871                                 vpnIntfId.augmentation(Adjacencies.class);
1872                             Optional<Adjacencies> adjacencies =
1873                                 VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
1874                             LOG.trace("adjacencies {}", adjacencies);
1875                             if (adjacencies.isPresent()) {
1876                                 List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
1877                                 Iterator<Adjacency> adjacencyIterator = adjacencyList.iterator();
1878
1879                                 while (adjacencyIterator.hasNext()) {
1880                                     Adjacency adjacency = adjacencyIterator.next();
1881                                     try {
1882                                         if (action == UpdateRouteAction.ADVERTISE_ROUTE) {
1883                                             LOG.info("VPNInterfaceManager : Added Fib Entry rd {} prefix {} nextHop {} label {}",
1884                                                      rd, adjacency.getIpAddress(), adjacency.getNextHopIpList(),
1885                                                      adjacency.getLabel());
1886 //                                            vrf = new VrfEntryBuilder().set
1887                                             if (tunTypeVal == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
1888                                                 fibManager.handleRemoteRoute(true,
1889                                                         new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId()),
1890                                                         new BigInteger(stateTunnelList.getDstInfo().getTepDeviceId()),
1891                                                         VpnUtil.getVpnId(broker, vpnInstance.getVpnInstanceName()),
1892                                                         rd, adjacency.getIpAddress(), srcTepIp, destTepIp);
1893                                             }
1894                                             if (tunTypeVal == VpnConstants.ITMTunnelLocType.External.getValue()) {
1895                                                 fibManager.populateFibOnDpn(srcDpnId, vpnId, rd, srcTepIp, destTepIp);
1896                                             }
1897                                         }
1898                                         else if (action == UpdateRouteAction.WITHDRAW_ROUTE) {
1899                                             LOG.info("VPNInterfaceManager : Removed Fib entry rd {} prefix {}",
1900                                                      rd, adjacency.getIpAddress());
1901                                             if (tunTypeVal == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
1902                                                 fibManager.handleRemoteRoute(false, srcDpnId,
1903                                                                 new BigInteger(stateTunnelList.getDstInfo().getTepDeviceId()),
1904                                                                 vpnId, rd, adjacency.getIpAddress(), srcTepIp, destTepIp);
1905                                             }
1906                                             if ((tunTypeVal == VpnConstants.ITMTunnelLocType.External.getValue()) &&
1907                                                     (dcgwPresentStatus == VpnConstants.DCGWPresentStatus.Absent.getValue())) {
1908                                                 bgpManager.withdrawPrefix(rd, adjacency.getIpAddress());
1909                                                 fibManager.cleanUpDpnForVpn(srcDpnId, vpnId, rd, srcTepIp, destTepIp);
1910                                             }
1911                                         }
1912                                     } catch (Exception e) {
1913                                         LOG.error("Exception when updating prefix {} in vrf {} to BGP",
1914                                             adjacency.getIpAddress(), rd);
1915                                     }
1916                                 }
1917                             } else {
1918                                 LOG.trace("no adjacencies present for path {}.", path);
1919                             }
1920
1921                         }
1922                         // if (action == UpdateRouteAction.WITHDRAW_ROUTE) {
1923                         //    fibManager.cleanUpDpnForVpn(dpnId, VpnUtil.getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
1924                         // }
1925                         // Go through all the VrfEntries and withdraw and readvertise the prefixes to BGP for which the nextHop is the SrcTepIp
1926                         if ((action == UpdateRouteAction.ADVERTISE_ROUTE) &&
1927                                 (tunTypeVal == VpnConstants.ITMTunnelLocType.External.getValue())) {
1928                             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(broker, rd);
1929                             if (vrfEntries != null) {
1930                                 for (VrfEntry vrfEntry : vrfEntries) {
1931                                     String destPrefix = vrfEntry.getDestPrefix().trim();
1932                                     int vpnLabel = vrfEntry.getLabel().intValue();
1933                                     List<String> nextHops = vrfEntry.getNextHopAddressList();
1934                                     if (nextHops.contains(srcTepIp.trim())) {
1935                                         bgpManager.withdrawPrefix(rd, destPrefix);
1936                                         bgpManager.advertisePrefix(rd, destPrefix, nextHops, vpnLabel);
1937                                     }
1938                                 }
1939                             }
1940                         }
1941                     } else {
1942                         LOG.trace("dpnInVpn check failed for srcDpnId {}.", srcDpnId);
1943                     }
1944                 } catch (Exception e) {
1945                     LOG.error("updatePrefixesForDPN {} in vpn {} failed", 0, vpnInstance.getVpnInstanceName(), e);
1946                 }
1947             }
1948         } else {
1949             LOG.trace("No vpn instances present.");
1950         }
1951     }
1952
1953     void publishAddNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1954         LOG.debug("Sending notification for add dpn {} in vpn {} event ", dpnId, vpnName);
1955         AddEventData data = new AddEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1956         AddDpnEvent event = new AddDpnEventBuilder().setAddEventData(data).build();
1957         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1958         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1959             @Override
1960             public void onFailure(Throwable error) {
1961                 LOG.warn("Error in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName, error);
1962             }
1963
1964             @Override
1965             public void onSuccess(Object arg) {
1966                 LOG.trace("Successful in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName);
1967             }
1968         });
1969     }
1970
1971     void publishRemoveNotification(final BigInteger dpnId, final String vpnName, final String rd) {
1972         LOG.debug("Sending notification for remove dpn {} in vpn {} event ", dpnId, vpnName);
1973         RemoveEventData data = new RemoveEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
1974         RemoveDpnEvent event = new RemoveDpnEventBuilder().setRemoveEventData(data).build();
1975         final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
1976         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
1977             @Override
1978             public void onFailure(Throwable error) {
1979                 LOG.warn("Error in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName, error);
1980             }
1981
1982             @Override
1983             public void onSuccess(Object arg) {
1984                 LOG.trace("Successful in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName);
1985             }
1986         });
1987     }
1988
1989     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1990         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1991             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1992             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1993     }
1994
1995     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1996         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1997             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1998     }
1999
2000     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeTxn) {
2001         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
2002         if(dpId.equals(BigInteger.ZERO)) {
2003             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
2004             return;
2005         }
2006         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2007
2008         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
2009                 .OPERATIONAL, routerDpnListIdentifier);
2010         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
2011         if (optionalRouterDpnList.isPresent()) {
2012             writeTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2013                     RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
2014         } else {
2015             RouterDpnListBuilder builder = new RouterDpnListBuilder();
2016             builder.setRouterId(routerName);
2017             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
2018             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
2019             routerInterfaces.add(routerInterface);
2020             builder.setDpnVpninterfacesList(Arrays.asList(dpnVpnList.build()));
2021             writeTxn.merge(LogicalDatastoreType.OPERATIONAL,
2022                     getRouterId(routerName),
2023                     new RouterDpnListBuilder().setRouterId(routerName).build(), true);
2024         }
2025     }
2026
2027     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, WriteTransaction writeTxn) {
2028         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
2029         if(dpId.equals(BigInteger.ZERO)) {
2030             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
2031             return;
2032         }
2033         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2034         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
2035                 .OPERATIONAL, routerDpnListIdentifier);
2036         if (optionalRouterDpnList.isPresent()) {
2037             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
2038             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
2039
2040             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
2041                 if (routerInterfaces.isEmpty()) {
2042                     if (writeTxn != null) {
2043                         writeTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2044                     } else {
2045                         MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2046                     }
2047                 } else {
2048                     if (writeTxn != null) {
2049                         writeTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2050                                 RouterInterfaces.class,
2051                                 new RouterInterfacesKey(vpnInterfaceName)));
2052                     } else {
2053                         MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2054                                 RouterInterfaces.class,
2055                                 new RouterInterfacesKey(vpnInterfaceName)));
2056                     }
2057                 }
2058             }
2059         }
2060     }
2061
2062     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId, WriteTransaction writeTxn) {
2063         if(dpId.equals(BigInteger.ZERO)) {
2064             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
2065             return;
2066         }
2067         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2068         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
2069                 .OPERATIONAL, routerDpnListIdentifier);
2070         if (optionalRouterDpnList.isPresent()) {
2071             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
2072             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
2073             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
2074                 if (routerInterfaces.isEmpty()) {
2075                     writeTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2076                 } else {
2077                     writeTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2078                             RouterInterfaces.class,
2079                             new RouterInterfacesKey(vpnInterfaceName)));
2080                 }
2081             }
2082         }
2083     }
2084
2085     public void addMIPAdjacency(String vpnName,String vpnInterface, IpAddress prefix){
2086
2087         LOG.trace("Adding {} adjacency to VPN Interface {} ",prefix,vpnInterface);
2088         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
2089         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
2090         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
2091         String nextHopIpAddr = null;
2092         String nextHopMacAddress = null;
2093         String ip = prefix.getIpv4Address().getValue();
2094         if (adjacencies.isPresent()) {
2095             List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
2096             ip = VpnUtil.getIpPrefix(ip);
2097             for (Adjacency adjacs : adjacencyList) {
2098                 if (adjacs.getMacAddress() != null && !adjacs.getMacAddress().isEmpty()) {
2099                     nextHopIpAddr = adjacs.getIpAddress();
2100                     nextHopMacAddress = adjacs.getMacAddress();
2101                     break;
2102                 }
2103             }
2104             if (nextHopMacAddress != null && ip != null) {
2105                 synchronized (vpnInterface.intern()) {
2106                     String rd = VpnUtil.getVpnRd(broker, vpnName);
2107                     long label =
2108                             VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
2109                                     VpnUtil.getNextHopLabelKey((rd != null) ? rd : vpnName, ip));
2110                     String nextHopIp = nextHopIpAddr.split("/")[0];
2111                     Adjacency newAdj = new AdjacencyBuilder().setIpAddress(ip).setKey
2112                             (new AdjacencyKey(ip)).setNextHopIpList(Arrays.asList(nextHopIp)).build();
2113                     adjacencyList.add(newAdj);
2114                     Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
2115                     VpnInterface newVpnIntf = new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(vpnInterface)).
2116                             setName(vpnInterface).setVpnInstanceName(vpnName).addAugmentation(Adjacencies.class, aug).build();
2117                     VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vpnIfId, newVpnIntf);
2118                 }
2119                 LOG.debug(" Successfully stored subnetroute Adjacency into VpnInterface {}", vpnInterface);
2120             }
2121         }
2122
2123     }
2124
2125     public void removeMIPAdjacency(String vpnName, String vpnInterface, IpAddress prefix) {
2126         String ip = VpnUtil.getIpPrefix(prefix.getIpv4Address().getValue());
2127         LOG.trace("Removing {} adjacency from Old VPN Interface {} ",ip,vpnInterface);
2128         InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
2129         InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
2130         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
2131         if (adjacencies.isPresent()) {
2132             synchronized (vpnInterface.intern()) {
2133                 InstanceIdentifier<Adjacency> adjacencyIdentifier = InstanceIdentifier.builder(VpnInterfaces.class).
2134                         child(VpnInterface.class, new VpnInterfaceKey(vpnInterface)).augmentation(Adjacencies.class)
2135                         .child(Adjacency.class, new AdjacencyKey(ip)).build();
2136                 MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, adjacencyIdentifier);
2137             }
2138             LOG.trace("Successfully Deleted Adjacency into VpnInterface {}", vpnInterface);
2139         }
2140     }
2141
2142     class TunnelInterfaceStateListener extends AbstractDataChangeListener<StateTunnelList>  {
2143         public TunnelInterfaceStateListener(final DataBroker db, VpnInterfaceManager vpnIfMgr) {
2144             super(StateTunnelList.class);
2145         }
2146
2147
2148         @Override
2149         protected void remove(InstanceIdentifier<StateTunnelList> identifier, StateTunnelList del) {
2150             LOG.trace("Tunnel deletion---- {}", del);
2151             handlePrefixesForDPNs(del, UpdateRouteAction.WITHDRAW_ROUTE);
2152         }
2153
2154         @Override
2155         protected void update(InstanceIdentifier<StateTunnelList> identifier, StateTunnelList original, StateTunnelList update) {
2156             LOG.trace("Tunnel updation---- {}", update);
2157             LOG.trace("ITM Tunnel {} of type {} state event changed from :{} to :{}",
2158                     update.getTunnelInterfaceName(),
2159                     fibManager.getTransportTypeStr(update.getTransportType().toString()),
2160                     original.isTunnelState(), update.isTunnelState());
2161                 //withdraw all prefixes in all vpns for this dpn
2162             boolean isTunnelUp = update.isTunnelState();
2163                 handlePrefixesForDPNs(update,isTunnelUp ?  UpdateRouteAction.ADVERTISE_ROUTE :
2164                                                             UpdateRouteAction.WITHDRAW_ROUTE);
2165         }
2166
2167         @Override
2168         protected void add(InstanceIdentifier<StateTunnelList> identifier, StateTunnelList add) {
2169             LOG.trace("Tunnel addition---- {}", add);
2170
2171             if(!add.isTunnelState()) {
2172                 LOG.trace(  "Tunnel {} is not yet UP.",
2173                             add.getTunnelInterfaceName());
2174                 return;
2175             } else {
2176                 LOG.trace("ITM Tunnel ,type {} ,State is UP b/w src: {} and dest: {}",
2177                         fibManager.getTransportTypeStr(add.getTransportType().toString()),
2178                         add.getSrcInfo().getTepDeviceId(), add.getDstInfo().getTepDeviceId());
2179                 handlePrefixesForDPNs(add, UpdateRouteAction.ADVERTISE_ROUTE);
2180             }
2181         }
2182     }
2183
2184 }