vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/vpnservice/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.vpnservice;
   9
  10 import org.opendaylight.vpnservice.utilities.InterfaceUtils;
  11
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.JdkFutureAdapters;
  15
  16 import org.opendaylight.controller.md.sal.binding.api.*;
  17 import org.opendaylight.vpnservice.mdsalutil.*;
  18 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  19 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  20 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyKey;
  21 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  22 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  23 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
  24 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.OdlArputilService;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.SendArpResponseInput;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.SendArpResponseInputBuilder;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.servicebinding.rev151015.service.bindings.services.info.BoundServices;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.FibEntries;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTables;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesKey;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntry;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntryKey;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.IdManagerService;
  39
  40 import java.math.BigInteger;
  41 import java.util.Collection;
  42 import java.util.Iterator;
  43 import java.util.List;
  44 import java.util.ArrayList;
  45 import java.util.concurrent.*;
  46
  47 import com.google.common.base.Optional;
  48
  49 import org.opendaylight.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.fibmanager.api.IFibManager;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.interfacemgr.rpcs.rev151003.OdlInterfaceRpcService;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.itm.rpcs.rev151217.ItmRpcService;
  53 import org.opendaylight.vpnservice.mdsalutil.interfaces.IMdsalApiManager;
  54 import org.opendaylight.yangtools.concepts.ListenerRegistration;
  55 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  56 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
  57 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  58 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.Adjacency;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.Adjacencies;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  69 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  70 import org.opendaylight.yangtools.yang.common.RpcError;
  71 import org.opendaylight.yangtools.yang.common.RpcResult;
  72 import org.slf4j.Logger;
  73 import org.slf4j.LoggerFactory;
  74
  75 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
  76     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
  77     private ListenerRegistration<DataChangeListener> listenerRegistration, opListenerRegistration;
  78     private ConcurrentMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
  79     private ExecutorService executorService = Executors.newSingleThreadExecutor();
  80     private final DataBroker broker;
  81     private final IBgpManager bgpManager;
  82     private IFibManager fibManager;
  83     private IMdsalApiManager mdsalManager;
  84     private OdlInterfaceRpcService interfaceManager;
  85     private ItmRpcService itmProvider;
  86     private IdManagerService idManager;
  87     private OdlArputilService arpManager;
  88     private InterfaceStateChangeListener interfaceListener;
  89     private VpnInterfaceOpListener vpnInterfaceOpListener;
  90     private ArpNotificationHandler arpNotificationHandler;
  91     protected enum UpdateRouteAction {
  92         ADVERTISE_ROUTE, WITHDRAW_ROUTE
  93     }
  94     /**
  95      * Responsible for listening to data change related to VPN Interface
  96      * Bind VPN Service on the interface and informs the BGP service
  97      *
  98      * @param db - dataBroker service reference
  99      */
 100     public VpnInterfaceManager(final DataBroker db, final IBgpManager bgpManager, NotificationService notificationService) {
 101         super(VpnInterface.class);
 102         broker = db;
 103         this.bgpManager = bgpManager;
 104         interfaceListener = new InterfaceStateChangeListener(db, this);
 105         vpnInterfaceOpListener = new VpnInterfaceOpListener();
 106         arpNotificationHandler = new ArpNotificationHandler(this, broker);
 107         notificationService.registerNotificationListener(arpNotificationHandler);
 108         registerListener(db);
 109     }
 110
 111     public void setMdsalManager(IMdsalApiManager mdsalManager) {
 112         this.mdsalManager = mdsalManager;
 113     }
 114
 115     public void setInterfaceManager(OdlInterfaceRpcService interfaceManager) {
 116         this.interfaceManager = interfaceManager;
 117         interfaceListener.setInterfaceManager(interfaceManager);
 118     }
 119
 120     public void setITMProvider(ItmRpcService itmProvider) {
 121         this.itmProvider = itmProvider;
 122     }
 123
 124     public void setFibManager(IFibManager fibManager) {
 125         this.fibManager = fibManager;
 126     }
 127
 128     public void setIdManager(IdManagerService idManager) {
 129         this.idManager = idManager;
 130     }
 131
 132     public void setArpManager(OdlArputilService arpManager) {
 133         this.arpManager = arpManager;
 134     }
 135
 136     @Override
 137     public void close() throws Exception {
 138         if (listenerRegistration != null) {
 139             try {
 140                 listenerRegistration.close();
 141                 opListenerRegistration.close();
 142             } catch (final Exception e) {
 143                 LOG.error("Error when cleaning up DataChangeListener.", e);
 144             }
 145             listenerRegistration = null;
 146             opListenerRegistration = null;
 147         }
 148         LOG.info("VPN Interface Manager Closed");
 149     }
 150
 151     private void registerListener(final DataBroker db) {
 152         try {
 153             listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 154                     getWildCardPath(), VpnInterfaceManager.this, DataChangeScope.SUBTREE);
 155             opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
 156                                                                    getWildCardPath(), vpnInterfaceOpListener, DataChangeScope.SUBTREE);
 157         } catch (final Exception e) {
 158             LOG.error("VPN Service DataChange listener registration fail!", e);
 159             throw new IllegalStateException("VPN Service registration Listener failed.", e);
 160         }
 161     }
 162
 163     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 164         return InstanceIdentifier.create(InterfacesState.class)
 165             .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 166     }
 167
 168     @Override
 169     protected void add(final InstanceIdentifier<VpnInterface> identifier,
 170             final VpnInterface vpnInterface) {
 171         LOG.trace("VPN Interface key: {} , value: {}", identifier, vpnInterface );
 172         addInterface(identifier, vpnInterface);
 173     }
 174
 175     private void addInterface(final InstanceIdentifier<VpnInterface> identifier,
 176                               final VpnInterface vpnInterface) {
 177         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 178         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 179         String interfaceName = key.getName();
 180
 181         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 182             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 183         if (interfaceState != null) {
 184             // Interface state is up
 185             processVpnInterfaceUp(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName, interfaceState.getIfIndex());
 186         } else {
 187             LOG.trace("VPN interfaces are not yet operational.");
 188         }
 189     }
 190
 191     protected void processVpnInterfaceUp(BigInteger dpId, String interfaceName, int lPortTag) {
 192
 193         VpnInterface vpnInterface = VpnUtil.getConfiguredVpnInterface(broker, interfaceName);
 194         if(vpnInterface == null) {
 195             LOG.info("Unable to process add/up for interface {} as it is not configured", interfaceName);
 196             return;
 197         }
 198         String vpnName = vpnInterface.getVpnInstanceName();
 199         LOG.info("Binding vpn service to interface {} ", interfaceName);
 200         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 201         if (vpnId == VpnConstants.INVALID_ID) {
 202             LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 203             return;
 204         }
 205         if (VpnUtil.getOperationalVpnInterface(broker, vpnInterface.getName()) != null) {
 206             LOG.trace("VPN Interface already provisioned , bailing out from here.");
 207             return;
 208         }
 209         synchronized (interfaceName.intern()) {
 210
 211             bindService(dpId, vpnName, interfaceName, lPortTag);
 212             updateDpnDbs(vpnName, interfaceName, true);
 213             processVpnInterfaceAdjacencies(VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()), vpnInterface);
 214         }
 215
 216     }
 217
 218     private void updateDpnDbs(String vpnName, String interfaceName, boolean add) {
 219         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 220         BigInteger dpId = InterfaceUtils.getDpnForInterface(interfaceManager, interfaceName);
 221         if(!dpId.equals(BigInteger.ZERO)) {
 222             if(add)
 223                 updateMappingDbs(vpnId, dpId, interfaceName, vpnName);
 224             else
 225                 removeFromMappingDbs(vpnId, dpId, interfaceName, vpnName);
 226         }
 227
 228     }
 229
 230     private void bindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName, int lPortTag) {
 231         int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 232         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 233
 234         int instructionKey = 0;
 235         List<Instruction> instructions = new ArrayList<Instruction>();
 236
 237         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 238         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 239
 240         BoundServices
 241             serviceInfo =
 242             InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 243                                             VpnConstants.L3VPN_SERVICE_IDENTIFIER, priority,
 244                                             VpnConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 245         VpnUtil.syncWrite(broker, LogicalDatastoreType.CONFIGURATION,
 246                           InterfaceUtils.buildServiceId(vpnInterfaceName, VpnConstants.L3VPN_SERVICE_IDENTIFIER), serviceInfo);
 247         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 248                     vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW);
 249
 250     }
 251
 252     private void processVpnInterfaceAdjacencies(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 253         String intfName = intf.getName();
 254
 255         synchronized (intfName) {
 256             // Read NextHops
 257             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 258             Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
 259
 260             if (adjacencies.isPresent()) {
 261                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 262                 List<Adjacency> value = new ArrayList<>();
 263
 264                 // Get the rd of the vpn instance
 265                 String rd = getRouteDistinguisher(intf.getVpnInstanceName());
 266
 267                 BigInteger dpnId = InterfaceUtils.getDpnForInterface(interfaceManager, intfName);
 268                 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 269                 if (nextHopIp == null){
 270                     LOG.error("NextHop for interface {} is null", intfName);
 271                 }
 272
 273                 LOG.trace("NextHops are {}", nextHops);
 274                 for (Adjacency nextHop : nextHops) {
 275                     String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 276                     long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, VpnUtil
 277                             .getNextHopLabelKey((rd == null) ? intf.getVpnInstanceName() : rd, prefix));
 278                     String adjNextHop = nextHop.getNextHopIp();
 279                     value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIp((adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp)
 280                             .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 281                     if(nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 282                         VpnUtil.syncUpdate(
 283                                 broker,
 284                                 LogicalDatastoreType.OPERATIONAL,
 285                                 VpnUtil.getPrefixToInterfaceIdentifier(
 286                                         VpnUtil.getVpnId(broker, intf.getVpnInstanceName()), prefix),
 287                                 VpnUtil.getPrefixToInterface(dpnId, intf.getName(), prefix));
 288                     }
 289                 }
 290
 291                 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 292                 VpnInterface opInterface = VpnUtil.getVpnInterface(intfName, intf.getVpnInstanceName(), aug);
 293                 InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(intfName);
 294                 VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface);
 295                 for (Adjacency nextHop : aug.getAdjacency()) {
 296                     long label = nextHop.getLabel();
 297                     String adjNextHop = nextHop.getNextHopIp();
 298                     if (rd != null) {
 299                         addPrefixToBGP(rd, nextHop.getIpAddress(),
 300                                             (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp, label);
 301                     } else {
 302                         // ### add FIB route directly
 303                         addFibEntryToDS(intf.getVpnInstanceName(), nextHop.getIpAddress(),
 304                                             (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp, (int) label);
 305                     }
 306                 }
 307             }
 308         }
 309     }
 310
 311     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 312                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow){
 313         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 314         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, BigInteger.valueOf(vpnId));
 315         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 316                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 317
 318         // Matching Arp reply flows
 319         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 320         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 321                 metadata, metadataMask }));
 322
 323         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 324
 325         // Instruction to punt to controller
 326         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 327         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 328         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 329         instructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 330
 331         // Install the flow entry in L3_INTERFACE_TABLE
 332         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 333                     NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 334         FlowEntity flowEntity;
 335         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 336                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 337                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 338
 339         if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 340             LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 341             mdsalManager.installFlow(flowEntity);
 342         } else {
 343             LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 344             mdsalManager.removeFlow(flowEntity);
 345         }
 346     }
 347
 348     private String getRouteDistinguisher(String vpnName) {
 349         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 350                                       .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 351         Optional<VpnInstance> vpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 352         String rd = "";
 353         if(vpnInstance.isPresent()) {
 354             VpnInstance instance = vpnInstance.get();
 355             VpnAfConfig config = instance.getIpv4Family();
 356             rd = config.getRouteDistinguisher();
 357         }
 358         return rd;
 359     }
 360
 361     private synchronized void updateMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 362         String routeDistinguisher = getRouteDistinguisher(vpnName);
 363         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 364         InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 365         Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 366         org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 367             vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 368
 369         if (dpnInVpn.isPresent()) {
 370             VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 371                 org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance
 372                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 373                     new VpnInterfacesKey(intfName)), vpnInterface);
 374         } else {
 375             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 376                                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 377                                     VpnUtil.getVpnInstanceOpDataBuilder(rd, vpnId));
 378             VpnToDpnListBuilder vpnToDpnList = new VpnToDpnListBuilder().setDpnId(dpnId);
 379             List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 380                 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces =  new ArrayList<>();
 381             vpnInterfaces.add(vpnInterface);
 382             VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id,
 383                               vpnToDpnList.setVpnInterfaces(vpnInterfaces).build());
 384
 385             /**
 386              * FIXME: DC Gateway tunnel should be built dynamically
 387             //this is the first VM in this VPN on the DPN, may be a new DPN has come up,
 388             //if tunnel to DC GW does not exist, create it
 389             //if(!tunnelExists(dpnID, bgpManager.getDCGWIP()))
 390             String dcGW = bgpManager.getDCGwIP();
 391             if(dcGW != null && !dcGW.isEmpty())
 392             {
 393                 LOG.debug("Building tunnel from DPN {} to DC GW {}", dpnId, dcGW);
 394                 itmProvider.buildTunnelFromDPNToDCGW(dpnId, new IpAddress(dcGW.toCharArray()));
 395             }*/
 396             fibManager.populateFibOnNewDpn(dpnId, vpnId, (rd == null) ? vpnName : rd);
 397         }
 398     }
 399
 400     private synchronized void removeFromMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 401         //TODO: Delay 'DPN' removal so that other services can cleanup the entries for this dpn
 402         String rd = VpnUtil.getVpnRd(broker, vpnName);
 403         InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 404         Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 405         if (dpnInVpn.isPresent()) {
 406             List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 407                 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 408             org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 409                     currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 410
 411             if (vpnInterfaces.remove(currVpnInterface)) {
 412                 if (vpnInterfaces.isEmpty()) {
 413                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id, VpnUtil.DEFAULT_CALLBACK);
 414                     fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
 415                 } else {
 416                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 417                         org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 418                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 419                             new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
 420                 }
 421             }
 422         }
 423     }
 424
 425     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label) {
 426         try {
 427             bgpManager.addPrefix(rd, prefix, nextHopIp, (int)label);
 428         } catch(Exception e) {
 429             LOG.error("Add prefix failed", e);
 430         }
 431     }
 432
 433
 434     private InstanceIdentifier<VpnInterface> getWildCardPath() {
 435         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 436     }
 437
 438     @Override
 439     protected void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
 440         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
 441         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 442         String interfaceName = key.getName();
 443
 444         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 445         Optional<VpnInterface> existingVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, interfaceId);
 446         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 447             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 448
 449         if (existingVpnInterface.isPresent() && interfaceState != null) {
 450             processVpnInterfaceDown(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName, interfaceState.getIfIndex(), false);
 451         } else {
 452             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
 453         }
 454     }
 455
 456     protected void processVpnInterfaceDown(BigInteger dpId, String interfaceName, int lPortTag, boolean isInterfaceStateDown) {
 457         VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
 458         if(vpnInterface == null) {
 459             LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
 460             return;
 461         }
 462         String vpnName = vpnInterface.getVpnInstanceName();
 463         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 464
 465         synchronized (interfaceName.intern()) {
 466             removeAdjacenciesFromVpn(identifier, vpnInterface);
 467             LOG.info("Unbinding vpn service from interface {} ", interfaceName);
 468             unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown);
 469
 470             //wait till DCN for removal of vpn interface in operational DS arrives
 471             Runnable notifyTask = new VpnNotifyTask();
 472             synchronized (interfaceName.intern()) {
 473                 vpnIntfMap.put(interfaceName, notifyTask);
 474                 synchronized (notifyTask) {
 475                     try {
 476                         notifyTask.wait(VpnConstants.WAIT_TIME_IN_MILLISECONDS);
 477                     } catch (InterruptedException e) {
 478                     }
 479                 }
 480             }
 481
 482         }
 483     }
 484
 485     private void removeAdjacenciesFromVpn(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 486         //Read NextHops
 487         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 488         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 489
 490         String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
 491         if (adjacencies.isPresent()) {
 492             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 493
 494             if (!nextHops.isEmpty()) {
 495                 LOG.trace("NextHops are " + nextHops);
 496                 for (Adjacency nextHop : nextHops) {
 497                     VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 498                                       VpnUtil.getNextHopLabelKey(rd, nextHop.getIpAddress()));
 499                     /*VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
 500                                    VpnUtil.getPrefixToInterfaceIdentifier(
 501                                        VpnUtil.getVpnId(broker, intf.getVpnInstanceName()),
 502                                        nextHop.getIpAddress()),
 503                                    VpnUtil.DEFAULT_CALLBACK);*/
 504                     if (rd.equals(intf.getVpnInstanceName())) {
 505                         //this is an internal vpn - the rd is assigned to the vpn instance name;
 506                         //remove from FIB directly
 507                         removeFibEntryFromDS(intf.getVpnInstanceName(), nextHop.getIpAddress());
 508                     } else {
 509                         removePrefixFromBGP(rd, nextHop.getIpAddress());
 510                     }
 511                 }
 512             }
 513         }
 514     }
 515
 516
 517     private void unbindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName,
 518                                int lPortTag, boolean isInterfaceStateDown) {
 519         if (!isInterfaceStateDown) {
 520             VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION,
 521                            InterfaceUtils.buildServiceId(vpnInterfaceName,
 522                                                          VpnConstants.L3VPN_SERVICE_IDENTIFIER),
 523                            VpnUtil.DEFAULT_CALLBACK);
 524         }
 525         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 526         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 527                     vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW);
 528     }
 529
 530
 531     private void removePrefixFromBGP(String rd, String prefix) {
 532         try {
 533             bgpManager.deletePrefix(rd, prefix);
 534         } catch(Exception e) {
 535             LOG.error("Delete prefix failed", e);
 536         }
 537     }
 538
 539     @Override
 540     protected void update(InstanceIdentifier<VpnInterface> identifier,
 541                                    VpnInterface original, VpnInterface update) {
 542         LOG.trace("Update VPN Interface {} , original {}, update {}",
 543                                                   identifier, original, update);
 544         String vpnName = original.getVpnInstanceName();
 545
 546         boolean vpnNameChanged = false;
 547         String rd = getRouteDistinguisher(vpnName);
 548         String newRd = rd;
 549         String newVpnName = update.getVpnInstanceName();
 550         if(!vpnName.equals(newVpnName)) {
 551             //VPN for this interface got changed.
 552             //Remove the interface from old VPN and add it to new VPN
 553             newRd = getRouteDistinguisher(newVpnName);
 554             if(newRd.equals("")) {
 555                 LOG.warn("VPN Instance {} not found. Update operation aborted", newVpnName);
 556                 return;
 557             }
 558             vpnNameChanged = true;
 559             LOG.debug("New VPN Name for the interface {} is {}", newVpnName, original.getName());
 560         }
 561
 562         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
 563         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
 564         if(vpnNameChanged && newAdjs != null && !newAdjs.isEmpty()) {
 565             long label = VpnConstants.INVALID_ID;
 566             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 567             Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 568             if (adjacencies.isPresent()) {
 569                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 570                 for(Adjacency nextHop : nextHops) {
 571                     label = nextHop.getLabel();
 572                     if(label == VpnConstants.INVALID_ID) {
 573                         //Generate label using ID Manager
 574                         label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 575                                                     VpnUtil.getNextHopLabelKey(newRd, nextHop.getIpAddress()));
 576                     }
 577                     if (rd != null) {
 578                         removePrefixFromBGP(rd, nextHop.getIpAddress());
 579                     } else {
 580                         removeFibEntryFromDS(vpnName, nextHop.getIpAddress());
 581                     }
 582                     //updatePrefixToBGP(newRd, nextHop, nextHopIp, label);
 583                 }
 584                 processVpnInterfaceAdjacencies(identifier, update);
 585                 VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, update);
 586             }
 587         } else if (oldAdjs != newAdjs) {
 588             //handle both addition and removal of adjacencies
 589             //currently, new adjacency may be an extra route
 590             for (Adjacency adj : newAdjs) {
 591                 if (oldAdjs.contains(adj)) {
 592                     oldAdjs.remove(adj);
 593                 } else {
 594                     // add new adjacency - right now only extra route will hit this path
 595                     addNewAdjToVpnInterface(identifier, adj);
 596                 }
 597             }
 598
 599             for (Adjacency adj : oldAdjs) {
 600                 delAdjFromVpnInterface(identifier, adj);
 601             }
 602         }
 603         else {
 604             LOG.debug("No Update information is available for VPN Interface to proceed");
 605         }
 606
 607     }
 608
 609     public void processArpRequest(IpAddress srcIP, PhysAddress srcMac, IpAddress targetIP, String srcInterface){
 610         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
 611                                                                     .setIpaddress(srcIP).setSrcIpAddress(targetIP).setMacaddress(srcMac).build();
 612         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
 613         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
 614         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
 615             @Override
 616             public void onFailure(Throwable error) {
 617                 LOG.error("Error - {}", msgFormat, error);
 618             }
 619
 620             @Override
 621             public void onSuccess(RpcResult<Void> result) {
 622                 if(!result.isSuccessful()) {
 623                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
 624                 } else {
 625                     LOG.debug("Successful RPC Result - {}", msgFormat);
 626                 }
 627             }
 628         });
 629     }
 630
 631     private String getErrorText(Collection<RpcError> errors) {
 632         StringBuilder errorText = new StringBuilder();
 633         for(RpcError error : errors) {
 634             errorText.append(",").append(error.getErrorType()).append("-")
 635                      .append(error.getMessage());
 636         }
 637         return errorText.toString();
 638     }
 639
 640     private String getTunnelInterfaceFlowRef(BigInteger dpnId, short tableId, String ifName) {
 641         return new StringBuilder().append(dpnId).append(tableId).append(ifName).toString();
 642     }
 643
 644
 645
 646     public synchronized void addFibEntryToDS(String rd, String prefix,
 647             String nexthop, int label) {
 648
 649         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).
 650                     setNextHopAddress(nexthop).setLabel((long)label).build();
 651         LOG.debug("Created vrfEntry for {} nexthop {} label {}", prefix, nexthop, label);
 652
 653         List<VrfEntry> vrfEntryList = new ArrayList<VrfEntry>();
 654         vrfEntryList.add(vrfEntry);
 655
 656         InstanceIdentifierBuilder<VrfTables> idBuilder =
 657                     InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
 658         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
 659
 660         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
 661                     setVrfEntry(vrfEntryList).build();
 662
 663         VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
 664     }
 665
 666     public synchronized void removeFibEntryFromDS(String rd, String prefix) {
 667
 668         LOG.debug("Removing fib entry with destination prefix {} from vrf table for rd {}", prefix, rd);
 669
 670         InstanceIdentifierBuilder<VrfEntry> idBuilder =
 671             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).child(VrfEntry.class, new VrfEntryKey(prefix));
 672         InstanceIdentifier<VrfEntry> vrfEntryId = idBuilder.build();
 673         VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId, VpnUtil.DEFAULT_CALLBACK);
 674
 675     }
 676
 677     public synchronized void removeVrfFromDS(String rd) {
 678         LOG.debug("Removing vrf table for  rd {}", rd);
 679
 680         InstanceIdentifierBuilder<VrfTables> idBuilder =
 681                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
 682         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
 683
 684         VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, VpnUtil.DEFAULT_CALLBACK);
 685
 686     }
 687
 688     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
 689
 690         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
 691
 692         if (optVpnInterface.isPresent()) {
 693             VpnInterface currVpnIntf = optVpnInterface.get();
 694             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
 695             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
 696             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
 697             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, adjPath);
 698             long label =
 699                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 700                             VpnUtil.getNextHopLabelKey((rd != null) ? rd : currVpnIntf.getVpnInstanceName(), prefix));
 701
 702             List<Adjacency> adjacencies;
 703             if (optAdjacencies.isPresent()) {
 704                 adjacencies = optAdjacencies.get().getAdjacency();
 705             } else {
 706                 //This code will not be hit since VM adjacency will always be there
 707                 adjacencies = new ArrayList<>();
 708             }
 709
 710             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIp(adj.getNextHopIp())
 711                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 712
 713             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
 714             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug);
 715
 716             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
 717             addExtraRoute(adj.getIpAddress(), adj.getNextHopIp(), rd, currVpnIntf.getVpnInstanceName(), (int) label, currVpnIntf.getName());
 718
 719         }
 720
 721     }
 722
 723     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
 724         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
 725
 726         if (optVpnInterface.isPresent()) {
 727             VpnInterface currVpnIntf = optVpnInterface.get();
 728
 729             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 730             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 731             if (optAdjacencies.isPresent()) {
 732                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 733
 734                 if (!adjacencies.isEmpty()) {
 735                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
 736                     LOG.trace("Adjacencies are " + adjacencies);
 737                     Iterator<Adjacency> adjIt = adjacencies.iterator();
 738                     while (adjIt.hasNext()) {
 739                         Adjacency adjElem = adjIt.next();
 740                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
 741                             VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 742                                     VpnUtil.getNextHopLabelKey(rd, adj.getIpAddress()));
 743                             adjIt.remove();
 744
 745                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
 746                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug);
 747
 748                             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
 749
 750                             delExtraRoute(adj.getIpAddress(), rd, currVpnIntf.getVpnInstanceName());
 751                             break;
 752                         }
 753
 754                     }
 755                 }
 756             }
 757         }
 758
 759     }
 760
 761     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label, String intfName) {
 762
 763         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 764         VpnUtil.syncUpdate(
 765             broker,
 766             LogicalDatastoreType.OPERATIONAL,
 767             VpnUtil.getVpnToExtrarouteIdentifier(
 768                 (rd != null) ? rd : routerID, destination),
 769             VpnUtil.getVpnToExtraroute(destination, nextHop));
 770
 771         if(intfName != null && !intfName.isEmpty()) {
 772             BigInteger dpnId = InterfaceUtils.getDpnForInterface(interfaceManager, intfName);
 773             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 774             if (nextHopIp == null && !nextHopIp.isEmpty()) {
 775                 LOG.error("NextHop for interface {} is null. Failed adding extra route for prefix {}", intfName, destination);
 776                 return;
 777             }
 778             nextHop = nextHopIp;
 779         }
 780         if (rd != null) {
 781             addPrefixToBGP(rd, destination, nextHop, label);
 782         } else {
 783             // ### add FIB route directly
 784             addFibEntryToDS(routerID, destination, nextHop, label);
 785         }
 786     }
 787
 788     protected void delExtraRoute(String destination, String rd, String routerID) {
 789         if (rd != null) {
 790             removePrefixFromBGP(rd, destination);
 791         } else {
 792             // ### add FIB route directly
 793             removeFibEntryFromDS(routerID, destination);
 794         }
 795     }
 796
 797     class VpnInterfaceOpListener extends org.opendaylight.vpnservice.mdsalutil.AbstractDataChangeListener<VpnInterface> {
 798
 799         public VpnInterfaceOpListener() {
 800             super(VpnInterface.class);
 801         }
 802
 803         @Override
 804         protected void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface del) {
 805             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 806             String interfaceName = key.getName();
 807
 808             //increment the vpn interface count in Vpn Instance Op Data
 809             Long ifCnt = 0L;
 810             String rd = getRouteDistinguisher(del.getVpnInstanceName());
 811             if(rd.isEmpty()) rd = del.getVpnInstanceName();
 812             VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
 813             if(vpnInstOp != null && vpnInstOp.getVpnInterfaceCount() != null) {
 814                 ifCnt = vpnInstOp.getVpnInterfaceCount();
 815             }
 816
 817             LOG.trace("VpnInterfaceOpListener remove: interface name {} rd {} interface count in Vpn Op Instance {}", interfaceName, rd, ifCnt);
 818
 819             VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 820                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 821                     VpnUtil.updateIntfCntInVpnInstOpData(ifCnt - 1, rd), VpnUtil.DEFAULT_CALLBACK);
 822
 823             //TODO: Clean up the DPN List in Vpn Instance Op if ifCnt is zero
 824
 825             notifyTaskIfRequired(interfaceName);
 826         }
 827
 828         private void notifyTaskIfRequired(String intfName) {
 829             Runnable notifyTask = vpnIntfMap.remove(intfName);
 830             if (notifyTask == null) {
 831                 return;
 832             }
 833             executorService.execute(notifyTask);
 834         }
 835
 836         @Override
 837         protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
 838         }
 839
 840         @Override
 841         protected void add(InstanceIdentifier<VpnInterface> identifier, VpnInterface add) {
 842             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 843             String interfaceName = key.getName();
 844
 845             //increment the vpn interface count in Vpn Instance Op Data
 846             Long ifCnt = 0L;
 847             String rd = getRouteDistinguisher(add.getVpnInstanceName());
 848             if(rd.isEmpty()) rd = add.getVpnInstanceName();
 849             VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
 850             if(vpnInstOp != null &&  vpnInstOp.getVpnInterfaceCount() != null) {
 851                 ifCnt = vpnInstOp.getVpnInterfaceCount();
 852             }
 853
 854             LOG.trace("VpnInterfaceOpListener add: interface name {} rd {} interface count in Vpn Op Instance {}", interfaceName, rd, ifCnt);
 855
 856             VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 857                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 858                     VpnUtil.updateIntfCntInVpnInstOpData(ifCnt + 1, rd), VpnUtil.DEFAULT_CALLBACK);
 859
 860
 861         }
 862     }
 863
 864     protected void updatePrefixesForDPN(BigInteger dpnId, UpdateRouteAction action) {
 865
 866         InstanceIdentifierBuilder<VpnInstances> idBuilder = InstanceIdentifier.builder(VpnInstances.class);
 867         InstanceIdentifier<VpnInstances> vpnInstancesId = idBuilder.build();
 868         Optional<VpnInstances> vpnInstances = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnInstancesId);
 869
 870         if (vpnInstances.isPresent()) {
 871             List<VpnInstance> vpnInstanceList = vpnInstances.get().getVpnInstance();
 872             Iterator<VpnInstance> vpnInstIter = vpnInstanceList.iterator();
 873             while (vpnInstIter.hasNext()) {
 874                 VpnInstance vpnInstance = vpnInstIter.next();
 875                 VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 876                 String rd = vpnConfig.getRouteDistinguisher();
 877
 878                 InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 879                 Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 880                 if (dpnInVpn.isPresent()) {
 881                     if (action == UpdateRouteAction.ADVERTISE_ROUTE) {
 882                         fibManager.populateFibOnNewDpn(dpnId, VpnUtil
 883                             .getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
 884                     }
 885                     List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 886                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 887                     for(org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 888                         .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces vpnInterface : vpnInterfaces) {
 889                         InstanceIdentifier<VpnInterface> vpnIntfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getInterfaceName());
 890                         InstanceIdentifier<Adjacencies> path = vpnIntfId.augmentation(Adjacencies.class);
 891                         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 892
 893                         if (adjacencies.isPresent()) {
 894                             List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
 895                             Iterator<Adjacency> adjacencyIterator = adjacencyList.iterator();
 896
 897                             while (adjacencyIterator.hasNext()) {
 898                                 Adjacency adjacency = adjacencyIterator.next();
 899                                 try {
 900                                     if(action == UpdateRouteAction.ADVERTISE_ROUTE)
 901                                         bgpManager.addPrefix(rd, adjacency.getIpAddress(), adjacency.getNextHopIp(), adjacency.getLabel().intValue());
 902                                     else if(action == UpdateRouteAction.WITHDRAW_ROUTE)
 903                                         bgpManager.deletePrefix(rd, adjacency.getIpAddress());
 904                                 } catch (Exception e) {
 905                                     LOG.error("Exception when updating prefix {} in vrf {} to BGP", adjacency.getIpAddress(), rd);
 906                                 }
 907                             }
 908                         }
 909
 910                     }
 911                     if (action == UpdateRouteAction.WITHDRAW_ROUTE) {
 912                         fibManager.cleanUpDpnForVpn(dpnId, VpnUtil
 913                             .getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
 914                     }
 915                 }
 916             }
 917         }
 918     }
 919
 920 }