vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/vpnservice/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.vpnservice;
   9
  10 import org.opendaylight.vpnservice.utilities.InterfaceUtils;
  11
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.JdkFutureAdapters;
  15
  16 import org.opendaylight.controller.md.sal.binding.api.*;
  17 import org.opendaylight.vpnservice.mdsalutil.*;
  18 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  19 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.neutronvpn.rev150602.NeutronvpnService;
  20 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  21 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.PrefixToInterface;
  22 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyKey;
  23 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  24 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.OdlArputilService;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.SendArpResponseInput;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.SendArpResponseInputBuilder;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.servicebinding.rev151015.service.bindings.services.info.BoundServices;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.FibEntries;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTables;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesKey;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntry;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntryKey;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.IdManagerService;
  42
  43 import java.math.BigInteger;
  44 import java.util.Collection;
  45 import java.util.Iterator;
  46 import java.util.List;
  47 import java.util.ArrayList;
  48 import java.util.concurrent.*;
  49
  50 import com.google.common.base.Optional;
  51
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.neutronvpn.rev150602.NeutronvpnService;
  53 import org.opendaylight.bgpmanager.api.IBgpManager;
  54 import org.opendaylight.fibmanager.api.IFibManager;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.interfacemgr.rpcs.rev151003.OdlInterfaceRpcService;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.itm.rpcs.rev151217.ItmRpcService;
  57 import org.opendaylight.vpnservice.mdsalutil.interfaces.IMdsalApiManager;
  58 import org.opendaylight.yangtools.concepts.ListenerRegistration;
  59 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  60 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
  61 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  62 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.Adjacency;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.Adjacencies;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  69 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  70 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  71 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  72 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  73 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  74 import org.opendaylight.yangtools.yang.common.RpcError;
  75 import org.opendaylight.yangtools.yang.common.RpcResult;
  76 import org.slf4j.Logger;
  77 import org.slf4j.LoggerFactory;
  78
  79 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
  80     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
  81     private ListenerRegistration<DataChangeListener> listenerRegistration, opListenerRegistration;
  82     private ConcurrentMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
  83     private ExecutorService executorService = Executors.newSingleThreadExecutor();
  84     private final DataBroker broker;
  85     private final IBgpManager bgpManager;
  86     private IFibManager fibManager;
  87     private IMdsalApiManager mdsalManager;
  88     private OdlInterfaceRpcService interfaceManager;
  89     private ItmRpcService itmProvider;
  90     private IdManagerService idManager;
  91     private OdlArputilService arpManager;
  92     private NeutronvpnService neuService;
  93     private VpnSubnetRouteHandler vpnSubnetRouteHandler;
  94     private InterfaceStateChangeListener interfaceListener;
  95     private VpnInterfaceOpListener vpnInterfaceOpListener;
  96     private ArpNotificationHandler arpNotificationHandler;
  97     protected enum UpdateRouteAction {
  98         ADVERTISE_ROUTE, WITHDRAW_ROUTE
  99     }
 100     /**
 101      * Responsible for listening to data change related to VPN Interface
 102      * Bind VPN Service on the interface and informs the BGP service
 103      *
 104      * @param db - dataBroker service reference
 105      */
 106     public VpnInterfaceManager(final DataBroker db, final IBgpManager bgpManager, NotificationService notificationService) {
 107         super(VpnInterface.class);
 108         broker = db;
 109         this.bgpManager = bgpManager;
 110         interfaceListener = new InterfaceStateChangeListener(db, this);
 111         vpnInterfaceOpListener = new VpnInterfaceOpListener();
 112         arpNotificationHandler = new ArpNotificationHandler(this, broker);
 113         notificationService.registerNotificationListener(arpNotificationHandler);
 114         vpnSubnetRouteHandler = new VpnSubnetRouteHandler(broker, bgpManager, this);
 115         notificationService.registerNotificationListener(vpnSubnetRouteHandler);
 116         registerListener(db);
 117     }
 118
 119     public void setMdsalManager(IMdsalApiManager mdsalManager) {
 120         this.mdsalManager = mdsalManager;
 121     }
 122
 123     public void setInterfaceManager(OdlInterfaceRpcService interfaceManager) {
 124         this.interfaceManager = interfaceManager;
 125         interfaceListener.setInterfaceManager(interfaceManager);
 126     }
 127
 128     public void setITMProvider(ItmRpcService itmProvider) {
 129         this.itmProvider = itmProvider;
 130     }
 131
 132     public void setFibManager(IFibManager fibManager) {
 133         this.fibManager = fibManager;
 134     }
 135
 136     public void setIdManager(IdManagerService idManager) {
 137         this.idManager = idManager;
 138         vpnSubnetRouteHandler.setIdManager(idManager);
 139     }
 140
 141     public void setArpManager(OdlArputilService arpManager) {
 142         this.arpManager = arpManager;
 143     }
 144
 145     public void setNeutronvpnManager(NeutronvpnService neuService) { this.neuService = neuService; }
 146
 147     public VpnSubnetRouteHandler getVpnSubnetRouteHandler() {
 148         return this.vpnSubnetRouteHandler;
 149     }
 150
 151     @Override
 152     public void close() throws Exception {
 153         if (listenerRegistration != null) {
 154             try {
 155                 listenerRegistration.close();
 156                 opListenerRegistration.close();
 157             } catch (final Exception e) {
 158                 LOG.error("Error when cleaning up DataChangeListener.", e);
 159             }
 160             listenerRegistration = null;
 161             opListenerRegistration = null;
 162         }
 163         LOG.info("VPN Interface Manager Closed");
 164     }
 165
 166     private void registerListener(final DataBroker db) {
 167         try {
 168             listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 169                     getWildCardPath(), VpnInterfaceManager.this, DataChangeScope.SUBTREE);
 170             opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
 171                                                                    getWildCardPath(), vpnInterfaceOpListener, DataChangeScope.SUBTREE);
 172         } catch (final Exception e) {
 173             LOG.error("VPN Service DataChange listener registration fail!", e);
 174             throw new IllegalStateException("VPN Service registration Listener failed.", e);
 175         }
 176     }
 177
 178     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 179         return InstanceIdentifier.create(InterfacesState.class)
 180             .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 181     }
 182
 183     @Override
 184     protected void add(final InstanceIdentifier<VpnInterface> identifier,
 185             final VpnInterface vpnInterface) {
 186         LOG.trace("VPN Interface key: {} , value: {}", identifier, vpnInterface );
 187         addInterface(identifier, vpnInterface);
 188     }
 189
 190     private void addInterface(final InstanceIdentifier<VpnInterface> identifier,
 191                               final VpnInterface vpnInterface) {
 192         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 193         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 194         String interfaceName = key.getName();
 195
 196         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 197             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 198         if (interfaceState != null) {
 199             // Interface state is up
 200             processVpnInterfaceUp(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName, interfaceState.getIfIndex());
 201         } else {
 202             LOG.trace("VPN interfaces are not yet operational.");
 203         }
 204     }
 205
 206     protected void processVpnInterfaceUp(BigInteger dpId, String interfaceName, int lPortTag) {
 207
 208         VpnInterface vpnInterface = VpnUtil.getConfiguredVpnInterface(broker, interfaceName);
 209         if(vpnInterface == null) {
 210             LOG.info("Unable to process add/up for interface {} as it is not configured", interfaceName);
 211             return;
 212         }
 213         String vpnName = vpnInterface.getVpnInstanceName();
 214         LOG.info("Binding vpn service to interface {} ", interfaceName);
 215         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 216         if (vpnId == VpnConstants.INVALID_ID) {
 217             LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 218             return;
 219         }
 220         synchronized (interfaceName.intern()) {
 221             if (VpnUtil.getOperationalVpnInterface(broker, vpnInterface.getName()) != null) {
 222                 LOG.trace("VPN Interface already provisioned , bailing out from here.");
 223                 return;
 224             }
 225             bindService(dpId, vpnName, interfaceName, lPortTag);
 226             updateDpnDbs(dpId, vpnName, interfaceName, true);
 227             processVpnInterfaceAdjacencies(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()), vpnInterface);
 228         }
 229
 230     }
 231
 232     private void updateDpnDbs(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 233         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 234         if (dpId == null) {
 235             dpId = InterfaceUtils.getDpnForInterface(interfaceManager, interfaceName);
 236         }
 237         if(!dpId.equals(BigInteger.ZERO)) {
 238             if(add)
 239                 updateMappingDbs(vpnId, dpId, interfaceName, vpnName);
 240             else
 241                 removeFromMappingDbs(vpnId, dpId, interfaceName, vpnName);
 242         }
 243
 244     }
 245
 246     private void bindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName, int lPortTag) {
 247         int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 248         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 249
 250         int instructionKey = 0;
 251         List<Instruction> instructions = new ArrayList<Instruction>();
 252
 253         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 254         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 255
 256         BoundServices
 257             serviceInfo =
 258             InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 259                                             VpnConstants.L3VPN_SERVICE_IDENTIFIER, priority,
 260                                             VpnConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 261         VpnUtil.syncWrite(broker, LogicalDatastoreType.CONFIGURATION,
 262                           InterfaceUtils.buildServiceId(vpnInterfaceName, VpnConstants.L3VPN_SERVICE_IDENTIFIER), serviceInfo);
 263         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 264                     vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW);
 265         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 266                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.ADD_FLOW);
 267
 268     }
 269
 270     private void processVpnInterfaceAdjacencies(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 271         String intfName = intf.getName();
 272
 273         synchronized (intfName) {
 274             // Read NextHops
 275             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 276             Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
 277
 278             if (adjacencies.isPresent()) {
 279                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 280                 List<Adjacency> value = new ArrayList<>();
 281
 282                 // Get the rd of the vpn instance
 283                 String rd = getRouteDistinguisher(intf.getVpnInstanceName());
 284
 285                 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 286                 if (nextHopIp == null){
 287                     LOG.error("NextHop for interface {} is null", intfName);
 288                 }
 289
 290                 LOG.trace("NextHops are {}", nextHops);
 291                 for (Adjacency nextHop : nextHops) {
 292                     String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 293                     long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, VpnUtil
 294                             .getNextHopLabelKey((rd == null) ? intf.getVpnInstanceName() : rd, prefix));
 295                     String adjNextHop = nextHop.getNextHopIp();
 296                     value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIp((adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp)
 297                             .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 298                     if(nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 299                         VpnUtil.syncUpdate(
 300                                 broker,
 301                                 LogicalDatastoreType.OPERATIONAL,
 302                                 VpnUtil.getPrefixToInterfaceIdentifier(
 303                                         VpnUtil.getVpnId(broker, intf.getVpnInstanceName()), prefix),
 304                                 VpnUtil.getPrefixToInterface(dpnId, intf.getName(), prefix));
 305                     } else {
 306                         //Extra route adjacency
 307                         VpnUtil.syncUpdate(
 308                                 broker,
 309                                 LogicalDatastoreType.OPERATIONAL,
 310                                 VpnUtil.getVpnToExtrarouteIdentifier(
 311                                         (rd != null) ? rd : intf.getVpnInstanceName(), nextHop.getIpAddress()),
 312                                 VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIp()));
 313
 314                     }
 315                 }
 316
 317                 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 318                 VpnInterface opInterface = VpnUtil.getVpnInterface(intfName, intf.getVpnInstanceName(), aug);
 319                 InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(intfName);
 320                 VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface);
 321                 for (Adjacency nextHop : aug.getAdjacency()) {
 322                     long label = nextHop.getLabel();
 323                     //String adjNextHop = nextHop.getNextHopIp();
 324                     if (rd != null) {
 325                         addPrefixToBGP(rd, nextHop.getIpAddress(),
 326                                             nextHopIp, label);
 327                     } else {
 328                         // ### add FIB route directly
 329                         addFibEntryToDS(intf.getVpnInstanceName(), nextHop.getIpAddress(),
 330                                             nextHopIp, (int) label);
 331                     }
 332                 }
 333             }
 334         }
 335     }
 336
 337     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 338                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow){
 339         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 340         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, BigInteger.valueOf(vpnId));
 341         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 342                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 343
 344         // Matching Arp reply flows
 345         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 346         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 347                 metadata, metadataMask }));
 348
 349         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 350
 351         // Instruction to punt to controller
 352         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 353         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 354         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 355         instructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 356
 357         // Install the flow entry in L3_INTERFACE_TABLE
 358         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 359                     NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 360         FlowEntity flowEntity;
 361         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 362                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 363                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 364
 365         if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 366             LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 367             mdsalManager.installFlow(flowEntity);
 368         } else {
 369             LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 370             mdsalManager.removeFlow(flowEntity);
 371         }
 372     }
 373
 374     private String getRouteDistinguisher(String vpnName) {
 375         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 376                                       .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 377         Optional<VpnInstance> vpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 378         String rd = "";
 379         if(vpnInstance.isPresent()) {
 380             VpnInstance instance = vpnInstance.get();
 381             VpnAfConfig config = instance.getIpv4Family();
 382             rd = config.getRouteDistinguisher();
 383         }
 384         return rd;
 385     }
 386
 387     private synchronized void updateMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 388         String routeDistinguisher = getRouteDistinguisher(vpnName);
 389         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 390         InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 391         Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 392         org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 393             vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 394
 395         if (dpnInVpn.isPresent()) {
 396             VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 397                 org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance
 398                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 399                     new VpnInterfacesKey(intfName)), vpnInterface);
 400         } else {
 401             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 402                                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 403                                     VpnUtil.getVpnInstanceOpDataBuilder(rd, vpnId));
 404             VpnToDpnListBuilder vpnToDpnList = new VpnToDpnListBuilder().setDpnId(dpnId);
 405             List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 406                 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces =  new ArrayList<>();
 407             vpnInterfaces.add(vpnInterface);
 408             VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id,
 409                               vpnToDpnList.setVpnInterfaces(vpnInterfaces).build());
 410
 411             /**
 412              * FIXME: DC Gateway tunnel should be built dynamically
 413             //this is the first VM in this VPN on the DPN, may be a new DPN has come up,
 414             //if tunnel to DC GW does not exist, create it
 415             //if(!tunnelExists(dpnID, bgpManager.getDCGWIP()))
 416             String dcGW = bgpManager.getDCGwIP();
 417             if(dcGW != null && !dcGW.isEmpty())
 418             {
 419                 LOG.debug("Building tunnel from DPN {} to DC GW {}", dpnId, dcGW);
 420                 itmProvider.buildTunnelFromDPNToDCGW(dpnId, new IpAddress(dcGW.toCharArray()));
 421             }*/
 422             fibManager.populateFibOnNewDpn(dpnId, vpnId, (rd == null) ? vpnName : rd);
 423         }
 424     }
 425
 426     private synchronized void removeFromMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 427         //TODO: Delay 'DPN' removal so that other services can cleanup the entries for this dpn
 428         String rd = VpnUtil.getVpnRd(broker, vpnName);
 429         InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 430         Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 431         if (dpnInVpn.isPresent()) {
 432             List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 433                 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 434             org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 435                     currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 436
 437             if (vpnInterfaces.remove(currVpnInterface)) {
 438                 if (vpnInterfaces.isEmpty()) {
 439                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id, VpnUtil.DEFAULT_CALLBACK);
 440                     fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
 441                 } else {
 442                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 443                         org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 444                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 445                             new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
 446                 }
 447             }
 448         }
 449     }
 450
 451     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label) {
 452         try {
 453             bgpManager.addPrefix(rd, prefix, nextHopIp, (int)label);
 454         } catch(Exception e) {
 455             LOG.error("Add prefix failed", e);
 456         }
 457     }
 458
 459
 460     private InstanceIdentifier<VpnInterface> getWildCardPath() {
 461         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 462     }
 463
 464     @Override
 465     protected void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
 466         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
 467         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 468         String interfaceName = key.getName();
 469
 470         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 471         Optional<VpnInterface> existingVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, interfaceId);
 472         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 473             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 474
 475         if (existingVpnInterface.isPresent() && interfaceState != null) {
 476             processVpnInterfaceDown(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName, interfaceState.getIfIndex(), false);
 477         } else {
 478             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
 479         }
 480     }
 481
 482     protected void processVpnInterfaceDown(BigInteger dpId, String interfaceName, int lPortTag, boolean isInterfaceStateDown) {
 483         VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
 484         if(vpnInterface == null) {
 485             LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
 486             return;
 487         }
 488         String vpnName = vpnInterface.getVpnInstanceName();
 489         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 490
 491         synchronized (interfaceName.intern()) {
 492             removeAdjacenciesFromVpn(identifier, vpnInterface);
 493             LOG.info("Unbinding vpn service from interface {} ", interfaceName);
 494             unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown);
 495
 496             //wait till DCN for removal of vpn interface in operational DS arrives
 497             Runnable notifyTask = new VpnNotifyTask();
 498             synchronized (interfaceName.intern()) {
 499                 vpnIntfMap.put(interfaceName, notifyTask);
 500                 synchronized (notifyTask) {
 501                     try {
 502                         notifyTask.wait(VpnConstants.MIN_WAIT_TIME_IN_MILLISECONDS);
 503                     } catch (InterruptedException e) {
 504                     }
 505                 }
 506             }
 507
 508         }
 509     }
 510
 511     private void removeAdjacenciesFromVpn(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 512         //Read NextHops
 513         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 514         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 515
 516         String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
 517         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", intf.getName(),
 518                 intf.getVpnInstanceName(), rd);
 519         if (adjacencies.isPresent()) {
 520             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 521
 522             if (!nextHops.isEmpty()) {
 523                 LOG.trace("NextHops are " + nextHops);
 524                 for (Adjacency nextHop : nextHops) {
 525                     // Commenting the release of ID here as it will be released by FIB
 526                    /* VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 527                                       VpnUtil.getNextHopLabelKey(rd, nextHop.getIpAddress()));
 528                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
 529                                    VpnUtil.getPrefixToInterfaceIdentifier(
 530                                        VpnUtil.getVpnId(broker, intf.getVpnInstanceName()),
 531                                        nextHop.getIpAddress()),
 532                                    VpnUtil.DEFAULT_CALLBACK);*/
 533                     if (rd.equals(intf.getVpnInstanceName())) {
 534                         //this is an internal vpn - the rd is assigned to the vpn instance name;
 535                         //remove from FIB directly
 536                         removeFibEntryFromDS(intf.getVpnInstanceName(), nextHop.getIpAddress());
 537                     } else {
 538                         removePrefixFromBGP(rd, nextHop.getIpAddress());
 539                     }
 540                 }
 541             }
 542         }
 543     }
 544
 545
 546     private void unbindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName,
 547                                int lPortTag, boolean isInterfaceStateDown) {
 548         if (!isInterfaceStateDown) {
 549             VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION,
 550                            InterfaceUtils.buildServiceId(vpnInterfaceName,
 551                                                          VpnConstants.L3VPN_SERVICE_IDENTIFIER),
 552                            VpnUtil.DEFAULT_CALLBACK);
 553         }
 554         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 555         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 556                     vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW);
 557         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 558                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.DEL_FLOW);
 559     }
 560
 561
 562     private void removePrefixFromBGP(String rd, String prefix) {
 563         try {
 564             bgpManager.deletePrefix(rd, prefix);
 565         } catch(Exception e) {
 566             LOG.error("Delete prefix failed", e);
 567         }
 568     }
 569
 570     @Override
 571     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
 572         if (LOG.isTraceEnabled()) {
 573             LOG.trace("Updating VPN Interface : key " + identifier + ",  original value=" + original + ", update " +
 574                     "value=" + update);
 575         }
 576         String oldVpnName = original.getVpnInstanceName();
 577         String newVpnName = update.getVpnInstanceName();
 578         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
 579         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
 580         if (oldAdjs == null) {
 581             oldAdjs = new ArrayList<>();
 582         }
 583         if (newAdjs == null) {
 584             newAdjs = new ArrayList<>();
 585         }
 586         //handles switching between <internal VPN - external VPN>
 587         if (!oldVpnName.equals(newVpnName)) {
 588             remove(identifier, original);
 589             add(identifier, update);
 590         }
 591         //handle both addition and removal of adjacencies
 592         //currently, new adjacency may be an extra route
 593         if (!oldAdjs.equals(newAdjs)) {
 594             for (Adjacency adj : newAdjs) {
 595                 if (oldAdjs.contains(adj)) {
 596                     oldAdjs.remove(adj);
 597                 } else {
 598                     // add new adjacency - right now only extra route will hit this path
 599                     addNewAdjToVpnInterface(identifier, adj);
 600                 }
 601             }
 602             for (Adjacency adj : oldAdjs) {
 603                 delAdjFromVpnInterface(identifier, adj);
 604             }
 605         }
 606     }
 607
 608     public void processArpRequest(IpAddress srcIP, PhysAddress srcMac, IpAddress targetIP, String srcInterface){
 609         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
 610                                                                     .setIpaddress(srcIP).setSrcIpAddress(targetIP).setMacaddress(srcMac).build();
 611         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
 612         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
 613         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
 614             @Override
 615             public void onFailure(Throwable error) {
 616                 LOG.error("Error - {}", msgFormat, error);
 617             }
 618
 619             @Override
 620             public void onSuccess(RpcResult<Void> result) {
 621                 if(!result.isSuccessful()) {
 622                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
 623                 } else {
 624                     LOG.debug("Successful RPC Result - {}", msgFormat);
 625                 }
 626             }
 627         });
 628     }
 629
 630     private String getErrorText(Collection<RpcError> errors) {
 631         StringBuilder errorText = new StringBuilder();
 632         for(RpcError error : errors) {
 633             errorText.append(",").append(error.getErrorType()).append("-")
 634                      .append(error.getMessage());
 635         }
 636         return errorText.toString();
 637     }
 638
 639     private String getTunnelInterfaceFlowRef(BigInteger dpnId, short tableId, String ifName) {
 640         return new StringBuilder().append(dpnId).append(tableId).append(ifName).toString();
 641     }
 642
 643
 644
 645     public synchronized void addFibEntryToDS(String rd, String prefix,
 646             String nexthop, int label) {
 647
 648         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).
 649                     setNextHopAddress(nexthop).setLabel((long)label).build();
 650         LOG.debug("Created vrfEntry for {} nexthop {} label {}", prefix, nexthop, label);
 651
 652         List<VrfEntry> vrfEntryList = new ArrayList<VrfEntry>();
 653         vrfEntryList.add(vrfEntry);
 654
 655         InstanceIdentifierBuilder<VrfTables> idBuilder =
 656                     InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
 657         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
 658
 659         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
 660                     setVrfEntry(vrfEntryList).build();
 661
 662         VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
 663     }
 664
 665     public synchronized void removeFibEntryFromDS(String rd, String prefix) {
 666
 667         LOG.debug("Removing fib entry with destination prefix {} from vrf table for rd {}", prefix, rd);
 668
 669         InstanceIdentifierBuilder<VrfEntry> idBuilder =
 670             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).child(VrfEntry.class, new VrfEntryKey(prefix));
 671         InstanceIdentifier<VrfEntry> vrfEntryId = idBuilder.build();
 672         VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId, VpnUtil.DEFAULT_CALLBACK);
 673
 674     }
 675
 676     public synchronized void removeVrfFromDS(String rd) {
 677         LOG.debug("Removing vrf table for  rd {}", rd);
 678
 679         InstanceIdentifierBuilder<VrfTables> idBuilder =
 680                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
 681         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
 682
 683         VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, VpnUtil.DEFAULT_CALLBACK);
 684
 685     }
 686
 687     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
 688
 689         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
 690
 691         if (optVpnInterface.isPresent()) {
 692             VpnInterface currVpnIntf = optVpnInterface.get();
 693             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
 694             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
 695             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
 696             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, adjPath);
 697             long label =
 698                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 699                             VpnUtil.getNextHopLabelKey((rd != null) ? rd : currVpnIntf.getVpnInstanceName(), prefix));
 700
 701             List<Adjacency> adjacencies;
 702             if (optAdjacencies.isPresent()) {
 703                 adjacencies = optAdjacencies.get().getAdjacency();
 704             } else {
 705                 //This code will not be hit since VM adjacency will always be there
 706                 adjacencies = new ArrayList<>();
 707             }
 708
 709             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIp(adj.getNextHopIp())
 710                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 711
 712             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
 713             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug);
 714
 715             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
 716             addExtraRoute(adj.getIpAddress(), adj.getNextHopIp(), rd, currVpnIntf.getVpnInstanceName(), (int) label, currVpnIntf.getName());
 717
 718         }
 719
 720     }
 721
 722     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
 723         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
 724
 725         if (optVpnInterface.isPresent()) {
 726             VpnInterface currVpnIntf = optVpnInterface.get();
 727
 728             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 729             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 730             if (optAdjacencies.isPresent()) {
 731                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 732
 733                 if (!adjacencies.isEmpty()) {
 734                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
 735                     LOG.trace("Adjacencies are " + adjacencies);
 736                     Iterator<Adjacency> adjIt = adjacencies.iterator();
 737                     while (adjIt.hasNext()) {
 738                         Adjacency adjElem = adjIt.next();
 739                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
 740                             // Commenting the release of ID here as it will be released by FIB
 741                            /* VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 742                                     VpnUtil.getNextHopLabelKey(rd, adj.getIpAddress()));*/
 743                             adjIt.remove();
 744
 745                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
 746                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
 747                                                                               currVpnIntf.getVpnInstanceName(),
 748                                                                               aug);
 749
 750                             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
 751
 752                             delExtraRoute(adj.getIpAddress(), rd, currVpnIntf.getVpnInstanceName());
 753                             break;
 754                         }
 755
 756                     }
 757                 }
 758             }
 759         }
 760
 761     }
 762
 763     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label, String intfName) {
 764
 765         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 766         VpnUtil.syncUpdate(
 767             broker,
 768             LogicalDatastoreType.OPERATIONAL,
 769             VpnUtil.getVpnToExtrarouteIdentifier(
 770                 (rd != null) ? rd : routerID, destination),
 771             VpnUtil.getVpnToExtraroute(destination, nextHop));
 772
 773         if (intfName != null && !intfName.isEmpty()) {
 774             BigInteger dpnId = InterfaceUtils.getDpnForInterface(interfaceManager, intfName);
 775             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 776             if (nextHopIp == null || nextHopIp.isEmpty()) {
 777                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}", intfName, destination);
 778             }
 779             nextHop = nextHopIp;
 780         }
 781         if (rd != null) {
 782             addPrefixToBGP(rd, destination, nextHop, label);
 783         } else {
 784             // ### add FIB route directly
 785             addFibEntryToDS(routerID, destination, nextHop, label);
 786         }
 787     }
 788
 789     protected void delExtraRoute(String destination, String rd, String routerID) {
 790         if (rd != null) {
 791             removePrefixFromBGP(rd, destination);
 792         } else {
 793             // ### add FIB route directly
 794             removeFibEntryFromDS(routerID, destination);
 795         }
 796     }
 797
 798     class VpnInterfaceOpListener extends org.opendaylight.vpnservice.mdsalutil.AbstractDataChangeListener<VpnInterface> {
 799
 800         public VpnInterfaceOpListener() {
 801             super(VpnInterface.class);
 802         }
 803
 804         @Override
 805         protected void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface del) {
 806             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 807             String interfaceName = key.getName();
 808             String vpnName = del.getVpnInstanceName();
 809
 810             LOG.trace("VpnInterfaceOpListener removed: interface name {} vpnName {}", interfaceName, vpnName);
 811             //decrement the vpn interface count in Vpn Instance Op Data
 812             InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance>
 813                     id = VpnUtil.getVpnInstanceToVpnIdIdentifier(vpnName);
 814             Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance> vpnInstance
 815                     = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 816
 817             if (vpnInstance.isPresent()) {
 818                 String rd = null;
 819                 rd = vpnInstance.get().getVrfId();
 820                 //String rd = getRouteDistinguisher(del.getVpnInstanceName());
 821
 822                 VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
 823                 LOG.trace("VpnInterfaceOpListener removed: interface name {} rd {} vpnName {} in Vpn Op Instance {}",
 824                         interfaceName, rd, vpnName, vpnInstOp);
 825
 826                 if (vpnInstOp != null) {
 827                     // Vpn Interface removed => No more adjacencies from it.
 828                     // Hence clean up interface from vpn-dpn-interface list.
 829                     Adjacency adjacency = del.getAugmentation(Adjacencies.class).getAdjacency().get(0);
 830                     Optional<Prefixes> prefixToInterface = Optional.absent();
 831                     prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 832                             VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
 833                                     VpnUtil.getIpPrefix(adjacency.getIpAddress())));
 834                     if (!prefixToInterface.isPresent()) {
 835                         prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 836                                 VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
 837                                         VpnUtil.getIpPrefix(adjacency.getNextHopIp())));
 838                     }
 839                     if (prefixToInterface.isPresent()) {
 840                         VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
 841                                 VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
 842                                         prefixToInterface.get().getIpAddress()),
 843                                 VpnUtil.DEFAULT_CALLBACK);
 844                         updateDpnDbs(prefixToInterface.get().getDpnId(), del.getVpnInstanceName(), interfaceName, false);
 845                     }
 846                     Long ifCnt = 0L;
 847                     ifCnt = vpnInstOp.getVpnInterfaceCount();
 848                     LOG.trace("VpnInterfaceOpListener removed: interface name {} rd {} vpnName {} Intf count {}",
 849                             interfaceName, rd, vpnName, ifCnt);
 850                     if ((ifCnt != null) && (ifCnt > 0)) {
 851                         VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 852                                 VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 853                                 VpnUtil.updateIntfCntInVpnInstOpData(ifCnt - 1, rd), VpnUtil.DEFAULT_CALLBACK);
 854                     }
 855                 }
 856             } else {
 857                 LOG.error("rd not retrievable as vpninstancetovpnid for vpn {} is absent, trying rd as ", vpnName, vpnName);
 858             }
 859             notifyTaskIfRequired(interfaceName);
 860         }
 861
 862         private void notifyTaskIfRequired(String intfName) {
 863             Runnable notifyTask = vpnIntfMap.remove(intfName);
 864             if (notifyTask == null) {
 865                 return;
 866             }
 867             executorService.execute(notifyTask);
 868         }
 869
 870         @Override
 871         protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
 872         }
 873
 874         @Override
 875         protected void add(InstanceIdentifier<VpnInterface> identifier, VpnInterface add) {
 876             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 877             String interfaceName = key.getName();
 878
 879             //increment the vpn interface count in Vpn Instance Op Data
 880             Long ifCnt = 0L;
 881             String rd = getRouteDistinguisher(add.getVpnInstanceName());
 882             if(rd == null || rd.isEmpty()) rd = add.getVpnInstanceName();
 883             VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
 884             if(vpnInstOp != null &&  vpnInstOp.getVpnInterfaceCount() != null) {
 885                 ifCnt = vpnInstOp.getVpnInterfaceCount();
 886             }
 887
 888             LOG.trace("VpnInterfaceOpListener add: interface name {} rd {} interface count in Vpn Op Instance {}", interfaceName, rd, ifCnt);
 889
 890             VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 891                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 892                     VpnUtil.updateIntfCntInVpnInstOpData(ifCnt + 1, rd), VpnUtil.DEFAULT_CALLBACK);
 893
 894
 895         }
 896     }
 897
 898     protected void updatePrefixesForDPN(BigInteger dpnId, UpdateRouteAction action) {
 899
 900         LOG.info("Tunnel event triggered {} for Dpn:{} ", action.name(), dpnId);
 901         InstanceIdentifierBuilder<VpnInstances> idBuilder = InstanceIdentifier.builder(VpnInstances.class);
 902         InstanceIdentifier<VpnInstances> vpnInstancesId = idBuilder.build();
 903         Optional<VpnInstances> vpnInstances = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnInstancesId);
 904
 905         if (vpnInstances.isPresent()) {
 906             List<VpnInstance> vpnInstanceList = vpnInstances.get().getVpnInstance();
 907             Iterator<VpnInstance> vpnInstIter = vpnInstanceList.iterator();
 908             while (vpnInstIter.hasNext()) {
 909                 VpnInstance vpnInstance = vpnInstIter.next();
 910                 try {
 911                     VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 912                     String rd = vpnConfig.getRouteDistinguisher();
 913                     if (rd == null || rd.isEmpty()) {
 914                         rd = vpnInstance.getVpnInstanceName();
 915                     }
 916                     InstanceIdentifier<VpnToDpnList> id =
 917                         VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 918                     Optional<VpnToDpnList> dpnInVpn =
 919                         VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 920                     if (dpnInVpn.isPresent()) {
 921                         // if (action == UpdateRouteAction.ADVERTISE_ROUTE) {
 922                         //    fibManager.populateFibOnNewDpn(dpnId, VpnUtil
 923                         //        .getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
 924                         // }
 925                         List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 926                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces>
 927                             vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 928                         for (org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 929                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces vpnInterface : vpnInterfaces) {
 930                             InstanceIdentifier<VpnInterface> vpnIntfId =
 931                                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getInterfaceName());
 932                             InstanceIdentifier<Adjacencies> path =
 933                                 vpnIntfId.augmentation(Adjacencies.class);
 934                             Optional<Adjacencies> adjacencies =
 935                                 VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 936
 937                             if (adjacencies.isPresent()) {
 938                                 List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
 939                                 Iterator<Adjacency> adjacencyIterator = adjacencyList.iterator();
 940
 941                                 while (adjacencyIterator.hasNext()) {
 942                                     Adjacency adjacency = adjacencyIterator.next();
 943                                     try {
 944                                         if (action == UpdateRouteAction.ADVERTISE_ROUTE)
 945                                             bgpManager.addPrefix(rd, adjacency.getIpAddress(),
 946                                                                  adjacency.getNextHopIp(),
 947                                                                  adjacency.getLabel().intValue());
 948                                         else if (action == UpdateRouteAction.WITHDRAW_ROUTE)
 949                                             bgpManager.deletePrefix(rd, adjacency.getIpAddress());
 950                                     } catch (Exception e) {
 951                                         LOG.error("Exception when updating prefix {} in vrf {} to BGP",
 952                                             adjacency.getIpAddress(), rd);
 953                                     }
 954                                 }
 955                             }
 956
 957                         }
 958                         // if (action == UpdateRouteAction.WITHDRAW_ROUTE) {
 959                         //    fibManager.cleanUpDpnForVpn(dpnId, VpnUtil.getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
 960                         // }
 961                     }
 962                 } catch (Exception e) {
 963                     LOG.error("updatePrefixesForDPN {} in vpn {} failed", dpnId, vpnInstance.getVpnInstanceName(), e);
 964                 }
 965             }
 966         }
 967     }
 968
 969 }