vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/vpnservice/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.vpnservice;
   9
  10 import org.opendaylight.vpnservice.utilities.InterfaceUtils;
  11
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.JdkFutureAdapters;
  15
  16 import org.opendaylight.controller.md.sal.binding.api.*;
  17 import org.opendaylight.vpnservice.mdsalutil.*;
  18 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  19 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  20 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.PrefixToInterface;
  21 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyKey;
  22 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  23 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  24 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.OdlArputilService;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.SendArpResponseInput;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.SendArpResponseInputBuilder;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.servicebinding.rev151015.service.bindings.services.info.BoundServices;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.FibEntries;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTables;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesKey;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntry;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntryKey;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.IdManagerService;
  41
  42 import java.math.BigInteger;
  43 import java.util.Collection;
  44 import java.util.Iterator;
  45 import java.util.List;
  46 import java.util.ArrayList;
  47 import java.util.concurrent.*;
  48
  49 import com.google.common.base.Optional;
  50
  51 import org.opendaylight.bgpmanager.api.IBgpManager;
  52 import org.opendaylight.fibmanager.api.IFibManager;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.interfacemgr.rpcs.rev151003.OdlInterfaceRpcService;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.itm.rpcs.rev151217.ItmRpcService;
  55 import org.opendaylight.vpnservice.mdsalutil.interfaces.IMdsalApiManager;
  56 import org.opendaylight.yangtools.concepts.ListenerRegistration;
  57 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  58 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
  59 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  60 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.Adjacency;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.Adjacencies;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  69 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  70 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  71 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  72 import org.opendaylight.yangtools.yang.common.RpcError;
  73 import org.opendaylight.yangtools.yang.common.RpcResult;
  74 import org.slf4j.Logger;
  75 import org.slf4j.LoggerFactory;
  76
  77 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
  78     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
  79     private ListenerRegistration<DataChangeListener> listenerRegistration, opListenerRegistration;
  80     private ConcurrentMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
  81     private ExecutorService executorService = Executors.newSingleThreadExecutor();
  82     private final DataBroker broker;
  83     private final IBgpManager bgpManager;
  84     private IFibManager fibManager;
  85     private IMdsalApiManager mdsalManager;
  86     private OdlInterfaceRpcService interfaceManager;
  87     private ItmRpcService itmProvider;
  88     private IdManagerService idManager;
  89     private OdlArputilService arpManager;
  90     private InterfaceStateChangeListener interfaceListener;
  91     private VpnInterfaceOpListener vpnInterfaceOpListener;
  92     private ArpNotificationHandler arpNotificationHandler;
  93     protected enum UpdateRouteAction {
  94         ADVERTISE_ROUTE, WITHDRAW_ROUTE
  95     }
  96     /**
  97      * Responsible for listening to data change related to VPN Interface
  98      * Bind VPN Service on the interface and informs the BGP service
  99      *
 100      * @param db - dataBroker service reference
 101      */
 102     public VpnInterfaceManager(final DataBroker db, final IBgpManager bgpManager, NotificationService notificationService) {
 103         super(VpnInterface.class);
 104         broker = db;
 105         this.bgpManager = bgpManager;
 106         interfaceListener = new InterfaceStateChangeListener(db, this);
 107         vpnInterfaceOpListener = new VpnInterfaceOpListener();
 108         arpNotificationHandler = new ArpNotificationHandler(this, broker);
 109         notificationService.registerNotificationListener(arpNotificationHandler);
 110         registerListener(db);
 111     }
 112
 113     public void setMdsalManager(IMdsalApiManager mdsalManager) {
 114         this.mdsalManager = mdsalManager;
 115     }
 116
 117     public void setInterfaceManager(OdlInterfaceRpcService interfaceManager) {
 118         this.interfaceManager = interfaceManager;
 119         interfaceListener.setInterfaceManager(interfaceManager);
 120     }
 121
 122     public void setITMProvider(ItmRpcService itmProvider) {
 123         this.itmProvider = itmProvider;
 124     }
 125
 126     public void setFibManager(IFibManager fibManager) {
 127         this.fibManager = fibManager;
 128     }
 129
 130     public void setIdManager(IdManagerService idManager) {
 131         this.idManager = idManager;
 132     }
 133
 134     public void setArpManager(OdlArputilService arpManager) {
 135         this.arpManager = arpManager;
 136     }
 137
 138     @Override
 139     public void close() throws Exception {
 140         if (listenerRegistration != null) {
 141             try {
 142                 listenerRegistration.close();
 143                 opListenerRegistration.close();
 144             } catch (final Exception e) {
 145                 LOG.error("Error when cleaning up DataChangeListener.", e);
 146             }
 147             listenerRegistration = null;
 148             opListenerRegistration = null;
 149         }
 150         LOG.info("VPN Interface Manager Closed");
 151     }
 152
 153     private void registerListener(final DataBroker db) {
 154         try {
 155             listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 156                     getWildCardPath(), VpnInterfaceManager.this, DataChangeScope.SUBTREE);
 157             opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
 158                                                                    getWildCardPath(), vpnInterfaceOpListener, DataChangeScope.SUBTREE);
 159         } catch (final Exception e) {
 160             LOG.error("VPN Service DataChange listener registration fail!", e);
 161             throw new IllegalStateException("VPN Service registration Listener failed.", e);
 162         }
 163     }
 164
 165     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 166         return InstanceIdentifier.create(InterfacesState.class)
 167             .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 168     }
 169
 170     @Override
 171     protected void add(final InstanceIdentifier<VpnInterface> identifier,
 172             final VpnInterface vpnInterface) {
 173         LOG.trace("VPN Interface key: {} , value: {}", identifier, vpnInterface );
 174         addInterface(identifier, vpnInterface);
 175     }
 176
 177     private void addInterface(final InstanceIdentifier<VpnInterface> identifier,
 178                               final VpnInterface vpnInterface) {
 179         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 180         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 181         String interfaceName = key.getName();
 182
 183         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 184             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 185         if (interfaceState != null) {
 186             // Interface state is up
 187             processVpnInterfaceUp(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName, interfaceState.getIfIndex());
 188         } else {
 189             LOG.trace("VPN interfaces are not yet operational.");
 190         }
 191     }
 192
 193     protected void processVpnInterfaceUp(BigInteger dpId, String interfaceName, int lPortTag) {
 194
 195         VpnInterface vpnInterface = VpnUtil.getConfiguredVpnInterface(broker, interfaceName);
 196         if(vpnInterface == null) {
 197             LOG.info("Unable to process add/up for interface {} as it is not configured", interfaceName);
 198             return;
 199         }
 200         String vpnName = vpnInterface.getVpnInstanceName();
 201         LOG.info("Binding vpn service to interface {} ", interfaceName);
 202         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 203         if (vpnId == VpnConstants.INVALID_ID) {
 204             LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 205             return;
 206         }
 207         synchronized (interfaceName.intern()) {
 208             if (VpnUtil.getOperationalVpnInterface(broker, vpnInterface.getName()) != null) {
 209                 LOG.trace("VPN Interface already provisioned , bailing out from here.");
 210                 return;
 211             }
 212             bindService(dpId, vpnName, interfaceName, lPortTag);
 213             updateDpnDbs(dpId, vpnName, interfaceName, true);
 214             processVpnInterfaceAdjacencies(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()), vpnInterface);
 215         }
 216
 217     }
 218
 219     private void updateDpnDbs(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 220         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 221         if (dpId == null) {
 222             dpId = InterfaceUtils.getDpnForInterface(interfaceManager, interfaceName);
 223         }
 224         if(!dpId.equals(BigInteger.ZERO)) {
 225             if(add)
 226                 updateMappingDbs(vpnId, dpId, interfaceName, vpnName);
 227             else
 228                 removeFromMappingDbs(vpnId, dpId, interfaceName, vpnName);
 229         }
 230
 231     }
 232
 233     private void bindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName, int lPortTag) {
 234         int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 235         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 236
 237         int instructionKey = 0;
 238         List<Instruction> instructions = new ArrayList<Instruction>();
 239
 240         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 241         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 242
 243         BoundServices
 244             serviceInfo =
 245             InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 246                                             VpnConstants.L3VPN_SERVICE_IDENTIFIER, priority,
 247                                             VpnConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 248         VpnUtil.syncWrite(broker, LogicalDatastoreType.CONFIGURATION,
 249                           InterfaceUtils.buildServiceId(vpnInterfaceName, VpnConstants.L3VPN_SERVICE_IDENTIFIER), serviceInfo);
 250         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 251                     vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW);
 252
 253     }
 254
 255     private void processVpnInterfaceAdjacencies(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 256         String intfName = intf.getName();
 257
 258         synchronized (intfName) {
 259             // Read NextHops
 260             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 261             Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
 262
 263             if (adjacencies.isPresent()) {
 264                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 265                 List<Adjacency> value = new ArrayList<>();
 266
 267                 // Get the rd of the vpn instance
 268                 String rd = getRouteDistinguisher(intf.getVpnInstanceName());
 269
 270                 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 271                 if (nextHopIp == null){
 272                     LOG.error("NextHop for interface {} is null", intfName);
 273                 }
 274
 275                 LOG.trace("NextHops are {}", nextHops);
 276                 for (Adjacency nextHop : nextHops) {
 277                     String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 278                     long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, VpnUtil
 279                             .getNextHopLabelKey((rd == null) ? intf.getVpnInstanceName() : rd, prefix));
 280                     String adjNextHop = nextHop.getNextHopIp();
 281                     value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIp((adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp)
 282                             .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 283                     if(nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 284                         VpnUtil.syncUpdate(
 285                                 broker,
 286                                 LogicalDatastoreType.OPERATIONAL,
 287                                 VpnUtil.getPrefixToInterfaceIdentifier(
 288                                         VpnUtil.getVpnId(broker, intf.getVpnInstanceName()), prefix),
 289                                 VpnUtil.getPrefixToInterface(dpnId, intf.getName(), prefix));
 290                     }
 291                 }
 292
 293                 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 294                 VpnInterface opInterface = VpnUtil.getVpnInterface(intfName, intf.getVpnInstanceName(), aug);
 295                 InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(intfName);
 296                 VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface);
 297                 for (Adjacency nextHop : aug.getAdjacency()) {
 298                     long label = nextHop.getLabel();
 299                     String adjNextHop = nextHop.getNextHopIp();
 300                     if (rd != null) {
 301                         addPrefixToBGP(rd, nextHop.getIpAddress(),
 302                                             (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp, label);
 303                     } else {
 304                         // ### add FIB route directly
 305                         addFibEntryToDS(intf.getVpnInstanceName(), nextHop.getIpAddress(),
 306                                             (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp, (int) label);
 307                     }
 308                 }
 309             }
 310         }
 311     }
 312
 313     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 314                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow){
 315         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 316         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, BigInteger.valueOf(vpnId));
 317         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 318                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 319
 320         // Matching Arp reply flows
 321         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 322         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 323                 metadata, metadataMask }));
 324
 325         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 326
 327         // Instruction to punt to controller
 328         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 329         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 330         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 331         instructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 332
 333         // Install the flow entry in L3_INTERFACE_TABLE
 334         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 335                     NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 336         FlowEntity flowEntity;
 337         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 338                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 339                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 340
 341         if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 342             LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 343             mdsalManager.installFlow(flowEntity);
 344         } else {
 345             LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 346             mdsalManager.removeFlow(flowEntity);
 347         }
 348     }
 349
 350     private String getRouteDistinguisher(String vpnName) {
 351         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 352                                       .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 353         Optional<VpnInstance> vpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 354         String rd = "";
 355         if(vpnInstance.isPresent()) {
 356             VpnInstance instance = vpnInstance.get();
 357             VpnAfConfig config = instance.getIpv4Family();
 358             rd = config.getRouteDistinguisher();
 359         }
 360         return rd;
 361     }
 362
 363     private synchronized void updateMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 364         String routeDistinguisher = getRouteDistinguisher(vpnName);
 365         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 366         InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 367         Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 368         org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 369             vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 370
 371         if (dpnInVpn.isPresent()) {
 372             VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 373                 org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance
 374                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 375                     new VpnInterfacesKey(intfName)), vpnInterface);
 376         } else {
 377             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 378                                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 379                                     VpnUtil.getVpnInstanceOpDataBuilder(rd, vpnId));
 380             VpnToDpnListBuilder vpnToDpnList = new VpnToDpnListBuilder().setDpnId(dpnId);
 381             List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 382                 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces =  new ArrayList<>();
 383             vpnInterfaces.add(vpnInterface);
 384             VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id,
 385                               vpnToDpnList.setVpnInterfaces(vpnInterfaces).build());
 386
 387             /**
 388              * FIXME: DC Gateway tunnel should be built dynamically
 389             //this is the first VM in this VPN on the DPN, may be a new DPN has come up,
 390             //if tunnel to DC GW does not exist, create it
 391             //if(!tunnelExists(dpnID, bgpManager.getDCGWIP()))
 392             String dcGW = bgpManager.getDCGwIP();
 393             if(dcGW != null && !dcGW.isEmpty())
 394             {
 395                 LOG.debug("Building tunnel from DPN {} to DC GW {}", dpnId, dcGW);
 396                 itmProvider.buildTunnelFromDPNToDCGW(dpnId, new IpAddress(dcGW.toCharArray()));
 397             }*/
 398             fibManager.populateFibOnNewDpn(dpnId, vpnId, (rd == null) ? vpnName : rd);
 399         }
 400     }
 401
 402     private synchronized void removeFromMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 403         //TODO: Delay 'DPN' removal so that other services can cleanup the entries for this dpn
 404         String rd = VpnUtil.getVpnRd(broker, vpnName);
 405         InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 406         Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 407         if (dpnInVpn.isPresent()) {
 408             List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 409                 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 410             org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 411                     currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 412
 413             if (vpnInterfaces.remove(currVpnInterface)) {
 414                 if (vpnInterfaces.isEmpty()) {
 415                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id, VpnUtil.DEFAULT_CALLBACK);
 416                     fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
 417                 } else {
 418                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 419                         org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 420                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 421                             new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
 422                 }
 423             }
 424         }
 425     }
 426
 427     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label) {
 428         try {
 429             bgpManager.addPrefix(rd, prefix, nextHopIp, (int)label);
 430         } catch(Exception e) {
 431             LOG.error("Add prefix failed", e);
 432         }
 433     }
 434
 435
 436     private InstanceIdentifier<VpnInterface> getWildCardPath() {
 437         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 438     }
 439
 440     @Override
 441     protected void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
 442         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
 443         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 444         String interfaceName = key.getName();
 445
 446         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 447         Optional<VpnInterface> existingVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, interfaceId);
 448         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 449             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 450
 451         if (existingVpnInterface.isPresent() && interfaceState != null) {
 452             processVpnInterfaceDown(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName, interfaceState.getIfIndex(), false);
 453         } else {
 454             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
 455         }
 456     }
 457
 458     protected void processVpnInterfaceDown(BigInteger dpId, String interfaceName, int lPortTag, boolean isInterfaceStateDown) {
 459         VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
 460         if(vpnInterface == null) {
 461             LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
 462             return;
 463         }
 464         String vpnName = vpnInterface.getVpnInstanceName();
 465         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 466
 467         synchronized (interfaceName.intern()) {
 468             removeAdjacenciesFromVpn(identifier, vpnInterface);
 469             LOG.info("Unbinding vpn service from interface {} ", interfaceName);
 470             unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown);
 471
 472             //wait till DCN for removal of vpn interface in operational DS arrives
 473             Runnable notifyTask = new VpnNotifyTask();
 474             synchronized (interfaceName.intern()) {
 475                 vpnIntfMap.put(interfaceName, notifyTask);
 476                 synchronized (notifyTask) {
 477                     try {
 478                         notifyTask.wait(VpnConstants.WAIT_TIME_IN_MILLISECONDS);
 479                     } catch (InterruptedException e) {
 480                     }
 481                 }
 482             }
 483
 484         }
 485     }
 486
 487     private void removeAdjacenciesFromVpn(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 488         //Read NextHops
 489         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 490         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 491
 492         String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
 493         if (adjacencies.isPresent()) {
 494             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 495
 496             if (!nextHops.isEmpty()) {
 497                 LOG.trace("NextHops are " + nextHops);
 498                 for (Adjacency nextHop : nextHops) {
 499                     VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 500                                       VpnUtil.getNextHopLabelKey(rd, nextHop.getIpAddress()));
 501                     /*VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
 502                                    VpnUtil.getPrefixToInterfaceIdentifier(
 503                                        VpnUtil.getVpnId(broker, intf.getVpnInstanceName()),
 504                                        nextHop.getIpAddress()),
 505                                    VpnUtil.DEFAULT_CALLBACK);*/
 506                     if (rd.equals(intf.getVpnInstanceName())) {
 507                         //this is an internal vpn - the rd is assigned to the vpn instance name;
 508                         //remove from FIB directly
 509                         removeFibEntryFromDS(intf.getVpnInstanceName(), nextHop.getIpAddress());
 510                     } else {
 511                         removePrefixFromBGP(rd, nextHop.getIpAddress());
 512                     }
 513                 }
 514             }
 515         }
 516     }
 517
 518
 519     private void unbindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName,
 520                                int lPortTag, boolean isInterfaceStateDown) {
 521         if (!isInterfaceStateDown) {
 522             VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION,
 523                            InterfaceUtils.buildServiceId(vpnInterfaceName,
 524                                                          VpnConstants.L3VPN_SERVICE_IDENTIFIER),
 525                            VpnUtil.DEFAULT_CALLBACK);
 526         }
 527         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 528         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 529                     vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW);
 530     }
 531
 532
 533     private void removePrefixFromBGP(String rd, String prefix) {
 534         try {
 535             bgpManager.deletePrefix(rd, prefix);
 536         } catch(Exception e) {
 537             LOG.error("Delete prefix failed", e);
 538         }
 539     }
 540
 541     @Override
 542     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
 543         if (LOG.isTraceEnabled()) {
 544             LOG.trace("Updating VPN Interface : key " + identifier + ",  original value=" + original + ", update " +
 545                     "value=" + update);
 546         }
 547         String oldVpnName = original.getVpnInstanceName();
 548         String newVpnName = update.getVpnInstanceName();
 549         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
 550         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
 551         if (oldAdjs == null) {
 552             oldAdjs = new ArrayList<>();
 553         }
 554         if (newAdjs == null) {
 555             newAdjs = new ArrayList<>();
 556         }
 557         //handles switching between <internal VPN - external VPN>
 558         if (!oldVpnName.equals(newVpnName)) {
 559             remove(identifier, original);
 560             add(identifier, update);
 561         }
 562         //handle both addition and removal of adjacencies
 563         //currently, new adjacency may be an extra route
 564         if (!oldAdjs.equals(newAdjs)) {
 565             for (Adjacency adj : newAdjs) {
 566                 if (oldAdjs.contains(adj)) {
 567                     oldAdjs.remove(adj);
 568                 } else {
 569                     // add new adjacency - right now only extra route will hit this path
 570                     addNewAdjToVpnInterface(identifier, adj);
 571                 }
 572             }
 573             for (Adjacency adj : oldAdjs) {
 574                 delAdjFromVpnInterface(identifier, adj);
 575             }
 576         }
 577     }
 578
 579     public void processArpRequest(IpAddress srcIP, PhysAddress srcMac, IpAddress targetIP, String srcInterface){
 580         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
 581                                                                     .setIpaddress(srcIP).setSrcIpAddress(targetIP).setMacaddress(srcMac).build();
 582         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
 583         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
 584         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
 585             @Override
 586             public void onFailure(Throwable error) {
 587                 LOG.error("Error - {}", msgFormat, error);
 588             }
 589
 590             @Override
 591             public void onSuccess(RpcResult<Void> result) {
 592                 if(!result.isSuccessful()) {
 593                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
 594                 } else {
 595                     LOG.debug("Successful RPC Result - {}", msgFormat);
 596                 }
 597             }
 598         });
 599     }
 600
 601     private String getErrorText(Collection<RpcError> errors) {
 602         StringBuilder errorText = new StringBuilder();
 603         for(RpcError error : errors) {
 604             errorText.append(",").append(error.getErrorType()).append("-")
 605                      .append(error.getMessage());
 606         }
 607         return errorText.toString();
 608     }
 609
 610     private String getTunnelInterfaceFlowRef(BigInteger dpnId, short tableId, String ifName) {
 611         return new StringBuilder().append(dpnId).append(tableId).append(ifName).toString();
 612     }
 613
 614
 615
 616     public synchronized void addFibEntryToDS(String rd, String prefix,
 617             String nexthop, int label) {
 618
 619         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).
 620                     setNextHopAddress(nexthop).setLabel((long)label).build();
 621         LOG.debug("Created vrfEntry for {} nexthop {} label {}", prefix, nexthop, label);
 622
 623         List<VrfEntry> vrfEntryList = new ArrayList<VrfEntry>();
 624         vrfEntryList.add(vrfEntry);
 625
 626         InstanceIdentifierBuilder<VrfTables> idBuilder =
 627                     InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
 628         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
 629
 630         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
 631                     setVrfEntry(vrfEntryList).build();
 632
 633         VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
 634     }
 635
 636     public synchronized void removeFibEntryFromDS(String rd, String prefix) {
 637
 638         LOG.debug("Removing fib entry with destination prefix {} from vrf table for rd {}", prefix, rd);
 639
 640         InstanceIdentifierBuilder<VrfEntry> idBuilder =
 641             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).child(VrfEntry.class, new VrfEntryKey(prefix));
 642         InstanceIdentifier<VrfEntry> vrfEntryId = idBuilder.build();
 643         VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId, VpnUtil.DEFAULT_CALLBACK);
 644
 645     }
 646
 647     public synchronized void removeVrfFromDS(String rd) {
 648         LOG.debug("Removing vrf table for  rd {}", rd);
 649
 650         InstanceIdentifierBuilder<VrfTables> idBuilder =
 651                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
 652         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
 653
 654         VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, VpnUtil.DEFAULT_CALLBACK);
 655
 656     }
 657
 658     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
 659
 660         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
 661
 662         if (optVpnInterface.isPresent()) {
 663             VpnInterface currVpnIntf = optVpnInterface.get();
 664             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
 665             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
 666             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
 667             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, adjPath);
 668             long label =
 669                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 670                             VpnUtil.getNextHopLabelKey((rd != null) ? rd : currVpnIntf.getVpnInstanceName(), prefix));
 671
 672             List<Adjacency> adjacencies;
 673             if (optAdjacencies.isPresent()) {
 674                 adjacencies = optAdjacencies.get().getAdjacency();
 675             } else {
 676                 //This code will not be hit since VM adjacency will always be there
 677                 adjacencies = new ArrayList<>();
 678             }
 679
 680             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIp(adj.getNextHopIp())
 681                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 682
 683             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
 684             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug);
 685
 686             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
 687             addExtraRoute(adj.getIpAddress(), adj.getNextHopIp(), rd, currVpnIntf.getVpnInstanceName(), (int) label, currVpnIntf.getName());
 688
 689         }
 690
 691     }
 692
 693     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
 694         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
 695
 696         if (optVpnInterface.isPresent()) {
 697             VpnInterface currVpnIntf = optVpnInterface.get();
 698
 699             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 700             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 701             if (optAdjacencies.isPresent()) {
 702                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 703
 704                 if (!adjacencies.isEmpty()) {
 705                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
 706                     LOG.trace("Adjacencies are " + adjacencies);
 707                     Iterator<Adjacency> adjIt = adjacencies.iterator();
 708                     while (adjIt.hasNext()) {
 709                         Adjacency adjElem = adjIt.next();
 710                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
 711                             VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 712                                     VpnUtil.getNextHopLabelKey(rd, adj.getIpAddress()));
 713                             adjIt.remove();
 714
 715                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
 716                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
 717                                                                               currVpnIntf.getVpnInstanceName(),
 718                                                                               aug);
 719
 720                             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
 721
 722                             delExtraRoute(adj.getIpAddress(), rd, currVpnIntf.getVpnInstanceName());
 723                             break;
 724                         }
 725
 726                     }
 727                 }
 728             }
 729         }
 730
 731     }
 732
 733     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label, String intfName) {
 734
 735         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 736         VpnUtil.syncUpdate(
 737             broker,
 738             LogicalDatastoreType.OPERATIONAL,
 739             VpnUtil.getVpnToExtrarouteIdentifier(
 740                 (rd != null) ? rd : routerID, destination),
 741             VpnUtil.getVpnToExtraroute(destination, nextHop));
 742
 743         if(intfName != null && !intfName.isEmpty()) {
 744             BigInteger dpnId = InterfaceUtils.getDpnForInterface(interfaceManager, intfName);
 745             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 746             if (nextHopIp == null && !nextHopIp.isEmpty()) {
 747                 LOG.error("NextHop for interface {} is null. Adding extra route {} without nextHop", intfName,
 748                         destination);
 749             }
 750             nextHop = nextHopIp;
 751         }
 752         if (rd != null) {
 753             addPrefixToBGP(rd, destination, nextHop, label);
 754         } else {
 755             // ### add FIB route directly
 756             addFibEntryToDS(routerID, destination, nextHop, label);
 757         }
 758     }
 759
 760     protected void delExtraRoute(String destination, String rd, String routerID) {
 761         if (rd != null) {
 762             removePrefixFromBGP(rd, destination);
 763         } else {
 764             // ### add FIB route directly
 765             removeFibEntryFromDS(routerID, destination);
 766         }
 767     }
 768
 769     class VpnInterfaceOpListener extends org.opendaylight.vpnservice.mdsalutil.AbstractDataChangeListener<VpnInterface> {
 770
 771         public VpnInterfaceOpListener() {
 772             super(VpnInterface.class);
 773         }
 774
 775         @Override
 776         protected void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface del) {
 777             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 778             String interfaceName = key.getName();
 779
 780             //increment the vpn interface count in Vpn Instance Op Data
 781             Long ifCnt = 0L;
 782             String rd = getRouteDistinguisher(del.getVpnInstanceName());
 783             if(rd == null || rd.isEmpty()) rd = del.getVpnInstanceName();
 784             VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
 785             if(vpnInstOp != null && vpnInstOp.getVpnInterfaceCount() != null) {
 786                 ifCnt = vpnInstOp.getVpnInterfaceCount();
 787             }
 788
 789             LOG.trace("VpnInterfaceOpListener remove: interface name {} rd {} interface count in Vpn Op Instance {}", interfaceName, rd, ifCnt);
 790
 791             if(ifCnt != 0) {
 792                 VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 793                                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 794                                     VpnUtil.updateIntfCntInVpnInstOpData(ifCnt - 1, rd), VpnUtil.DEFAULT_CALLBACK);
 795             }
 796
 797             // Vpn Interface removed => No more adjacencies from it.
 798             // Hence clean up interface from vpn-dpn-interface list.
 799             Adjacency adjacency = del.getAugmentation(Adjacencies.class).getAdjacency().get(0);
 800             Optional<Prefixes> prefixToInterface = Optional.absent();
 801             prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 802                          VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
 803                                                 VpnUtil.getIpPrefix(adjacency.getIpAddress())));
 804             if (!prefixToInterface.isPresent()) {
 805                 prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 806                                                  VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
 807                                                          VpnUtil.getIpPrefix(adjacency.getNextHopIp())));
 808             }
 809             if (prefixToInterface.isPresent()) {
 810                 VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
 811                                VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
 812                                                  prefixToInterface.get().getIpAddress()),
 813                                VpnUtil.DEFAULT_CALLBACK);
 814                 updateDpnDbs(prefixToInterface.get().getDpnId(), del.getVpnInstanceName(), interfaceName, false);
 815             }
 816             notifyTaskIfRequired(interfaceName);
 817         }
 818
 819         private void notifyTaskIfRequired(String intfName) {
 820             Runnable notifyTask = vpnIntfMap.remove(intfName);
 821             if (notifyTask == null) {
 822                 return;
 823             }
 824             executorService.execute(notifyTask);
 825         }
 826
 827         @Override
 828         protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
 829         }
 830
 831         @Override
 832         protected void add(InstanceIdentifier<VpnInterface> identifier, VpnInterface add) {
 833             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 834             String interfaceName = key.getName();
 835
 836             //increment the vpn interface count in Vpn Instance Op Data
 837             Long ifCnt = 0L;
 838             String rd = getRouteDistinguisher(add.getVpnInstanceName());
 839             if(rd == null || rd.isEmpty()) rd = add.getVpnInstanceName();
 840             VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
 841             if(vpnInstOp != null &&  vpnInstOp.getVpnInterfaceCount() != null) {
 842                 ifCnt = vpnInstOp.getVpnInterfaceCount();
 843             }
 844
 845             LOG.trace("VpnInterfaceOpListener add: interface name {} rd {} interface count in Vpn Op Instance {}", interfaceName, rd, ifCnt);
 846
 847             VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 848                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 849                     VpnUtil.updateIntfCntInVpnInstOpData(ifCnt + 1, rd), VpnUtil.DEFAULT_CALLBACK);
 850
 851
 852         }
 853     }
 854
 855     protected void updatePrefixesForDPN(BigInteger dpnId, UpdateRouteAction action) {
 856
 857         LOG.info("Tunnel event triggered {} for Dpn:{} ", action.name(), dpnId);
 858         InstanceIdentifierBuilder<VpnInstances> idBuilder = InstanceIdentifier.builder(VpnInstances.class);
 859         InstanceIdentifier<VpnInstances> vpnInstancesId = idBuilder.build();
 860         Optional<VpnInstances> vpnInstances = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnInstancesId);
 861
 862         if (vpnInstances.isPresent()) {
 863             List<VpnInstance> vpnInstanceList = vpnInstances.get().getVpnInstance();
 864             Iterator<VpnInstance> vpnInstIter = vpnInstanceList.iterator();
 865             while (vpnInstIter.hasNext()) {
 866                 VpnInstance vpnInstance = vpnInstIter.next();
 867                 try {
 868                     VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 869                     String rd = vpnConfig.getRouteDistinguisher();
 870                     if (rd == null || rd.isEmpty()) {
 871                         rd = vpnInstance.getVpnInstanceName();
 872                     }
 873                     InstanceIdentifier<VpnToDpnList> id =
 874                         VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 875                     Optional<VpnToDpnList> dpnInVpn =
 876                         VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 877                     if (dpnInVpn.isPresent()) {
 878                         // if (action == UpdateRouteAction.ADVERTISE_ROUTE) {
 879                         //    fibManager.populateFibOnNewDpn(dpnId, VpnUtil
 880                         //        .getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
 881                         // }
 882                         List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 883                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces>
 884                             vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 885                         for (org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 886                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces vpnInterface : vpnInterfaces) {
 887                             InstanceIdentifier<VpnInterface> vpnIntfId =
 888                                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getInterfaceName());
 889                             InstanceIdentifier<Adjacencies> path =
 890                                 vpnIntfId.augmentation(Adjacencies.class);
 891                             Optional<Adjacencies> adjacencies =
 892                                 VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 893
 894                             if (adjacencies.isPresent()) {
 895                                 List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
 896                                 Iterator<Adjacency> adjacencyIterator = adjacencyList.iterator();
 897
 898                                 while (adjacencyIterator.hasNext()) {
 899                                     Adjacency adjacency = adjacencyIterator.next();
 900                                     try {
 901                                         if (action == UpdateRouteAction.ADVERTISE_ROUTE)
 902                                             bgpManager.addPrefix(rd, adjacency.getIpAddress(),
 903                                                                  adjacency.getNextHopIp(),
 904                                                                  adjacency.getLabel().intValue());
 905                                         else if (action == UpdateRouteAction.WITHDRAW_ROUTE)
 906                                             bgpManager.deletePrefix(rd, adjacency.getIpAddress());
 907                                     } catch (Exception e) {
 908                                         LOG.error("Exception when updating prefix {} in vrf {} to BGP",
 909                                             adjacency.getIpAddress(), rd);
 910                                     }
 911                                 }
 912                             }
 913
 914                         }
 915                         // if (action == UpdateRouteAction.WITHDRAW_ROUTE) {
 916                         //    fibManager.cleanUpDpnForVpn(dpnId, VpnUtil.getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
 917                         // }
 918                     }
 919                 } catch (Exception e) {
 920                     LOG.error("updatePrefixesForDPN {} in vpn {} failed", dpnId, vpnInstance.getVpnInstanceName(), e);
 921                 }
 922             }
 923         }
 924     }
 925
 926 }