vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/vpnservice/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.vpnservice;
   9
  10 import org.opendaylight.vpnservice.utilities.InterfaceUtils;
  11
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.JdkFutureAdapters;
  15
  16 import org.opendaylight.controller.md.sal.binding.api.*;
  17 import org.opendaylight.vpnservice.mdsalutil.*;
  18 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  19 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.NeutronRouterDpns;
  20 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.neutronvpn.rev150602.NeutronvpnService;
  21 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  22 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.PrefixToInterface;
  23 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyKey;
  24 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.OdlArputilService;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.SendArpResponseInput;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.SendArpResponseInputBuilder;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.servicebinding.rev151015.service.bindings.services.info.BoundServices;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.FibEntries;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTables;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesKey;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntry;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntryKey;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.IdManagerService;
  52
  53 import java.math.BigInteger;
  54 import java.util.Collection;
  55 import java.util.Iterator;
  56 import java.util.List;
  57 import java.util.ArrayList;
  58 import java.util.concurrent.*;
  59
  60 import com.google.common.base.Optional;
  61
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.neutronvpn.rev150602.NeutronvpnService;
  63 import org.opendaylight.bgpmanager.api.IBgpManager;
  64 import org.opendaylight.fibmanager.api.IFibManager;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.interfacemgr.rpcs.rev151003.OdlInterfaceRpcService;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.itm.rpcs.rev151217.ItmRpcService;
  67 import org.opendaylight.vpnservice.mdsalutil.interfaces.IMdsalApiManager;
  68 import org.opendaylight.yangtools.concepts.ListenerRegistration;
  69 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  70 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
  71 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  72 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  73 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.Adjacency;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.Adjacencies;
  78 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  79 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  80 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  81 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  82 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  83 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  84 import org.opendaylight.yangtools.yang.common.RpcError;
  85 import org.opendaylight.yangtools.yang.common.RpcResult;
  86 import org.slf4j.Logger;
  87 import org.slf4j.LoggerFactory;
  88
  89 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
  90     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
  91     private ListenerRegistration<DataChangeListener> listenerRegistration, opListenerRegistration;
  92     private ConcurrentMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
  93     private ExecutorService executorService = Executors.newSingleThreadExecutor();
  94     private final DataBroker broker;
  95     private final IBgpManager bgpManager;
  96     private IFibManager fibManager;
  97     private IMdsalApiManager mdsalManager;
  98     private OdlInterfaceRpcService interfaceManager;
  99     private ItmRpcService itmProvider;
 100     private IdManagerService idManager;
 101     private OdlArputilService arpManager;
 102     private NeutronvpnService neuService;
 103     private VpnSubnetRouteHandler vpnSubnetRouteHandler;
 104     private InterfaceStateChangeListener interfaceListener;
 105     private VpnInterfaceOpListener vpnInterfaceOpListener;
 106     private ArpNotificationHandler arpNotificationHandler;
 107     protected enum UpdateRouteAction {
 108         ADVERTISE_ROUTE, WITHDRAW_ROUTE
 109     }
 110     /**
 111      * Responsible for listening to data change related to VPN Interface
 112      * Bind VPN Service on the interface and informs the BGP service
 113      *
 114      * @param db - dataBroker service reference
 115      */
 116     public VpnInterfaceManager(final DataBroker db, final IBgpManager bgpManager, NotificationService notificationService) {
 117         super(VpnInterface.class);
 118         broker = db;
 119         this.bgpManager = bgpManager;
 120         interfaceListener = new InterfaceStateChangeListener(db, this);
 121         vpnInterfaceOpListener = new VpnInterfaceOpListener();
 122         arpNotificationHandler = new ArpNotificationHandler(this, broker);
 123         notificationService.registerNotificationListener(arpNotificationHandler);
 124         vpnSubnetRouteHandler = new VpnSubnetRouteHandler(broker, bgpManager, this);
 125         notificationService.registerNotificationListener(vpnSubnetRouteHandler);
 126         registerListener(db);
 127     }
 128
 129     public void setMdsalManager(IMdsalApiManager mdsalManager) {
 130         this.mdsalManager = mdsalManager;
 131     }
 132
 133     public void setInterfaceManager(OdlInterfaceRpcService interfaceManager) {
 134         this.interfaceManager = interfaceManager;
 135         interfaceListener.setInterfaceManager(interfaceManager);
 136     }
 137
 138     public void setITMProvider(ItmRpcService itmProvider) {
 139         this.itmProvider = itmProvider;
 140     }
 141
 142     public void setFibManager(IFibManager fibManager) {
 143         this.fibManager = fibManager;
 144     }
 145
 146     public void setIdManager(IdManagerService idManager) {
 147         this.idManager = idManager;
 148         vpnSubnetRouteHandler.setIdManager(idManager);
 149     }
 150
 151     public void setArpManager(OdlArputilService arpManager) {
 152         this.arpManager = arpManager;
 153     }
 154
 155     public void setNeutronvpnManager(NeutronvpnService neuService) { this.neuService = neuService; }
 156
 157     public VpnSubnetRouteHandler getVpnSubnetRouteHandler() {
 158         return this.vpnSubnetRouteHandler;
 159     }
 160
 161     @Override
 162     public void close() throws Exception {
 163         if (listenerRegistration != null) {
 164             try {
 165                 listenerRegistration.close();
 166                 opListenerRegistration.close();
 167             } catch (final Exception e) {
 168                 LOG.error("Error when cleaning up DataChangeListener.", e);
 169             }
 170             listenerRegistration = null;
 171             opListenerRegistration = null;
 172         }
 173         LOG.info("VPN Interface Manager Closed");
 174     }
 175
 176     private void registerListener(final DataBroker db) {
 177         try {
 178             listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 179                     getWildCardPath(), VpnInterfaceManager.this, DataChangeScope.SUBTREE);
 180             opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
 181                                                                    getWildCardPath(), vpnInterfaceOpListener, DataChangeScope.SUBTREE);
 182         } catch (final Exception e) {
 183             LOG.error("VPN Service DataChange listener registration fail!", e);
 184             throw new IllegalStateException("VPN Service registration Listener failed.", e);
 185         }
 186     }
 187
 188     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
 189         return InstanceIdentifier.create(InterfacesState.class)
 190             .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
 191     }
 192
 193     @Override
 194     protected void add(final InstanceIdentifier<VpnInterface> identifier,
 195             final VpnInterface vpnInterface) {
 196         LOG.trace("VPN Interface key: {} , value: {}", identifier, vpnInterface );
 197         addInterface(identifier, vpnInterface);
 198     }
 199
 200     private void addInterface(final InstanceIdentifier<VpnInterface> identifier,
 201                               final VpnInterface vpnInterface) {
 202         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
 203         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 204         String interfaceName = key.getName();
 205
 206         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 207             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 208         if (interfaceState != null) {
 209             // Interface state is up
 210             processVpnInterfaceUp(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName, interfaceState.getIfIndex());
 211         } else {
 212             LOG.trace("VPN interfaces are not yet operational.");
 213         }
 214     }
 215
 216     protected void processVpnInterfaceUp(BigInteger dpId, String interfaceName, int lPortTag) {
 217
 218         VpnInterface vpnInterface = VpnUtil.getConfiguredVpnInterface(broker, interfaceName);
 219         if(vpnInterface == null) {
 220             LOG.info("Unable to process add/up for interface {} as it is not configured", interfaceName);
 221             return;
 222         }
 223         String vpnName = vpnInterface.getVpnInstanceName();
 224         LOG.info("Binding vpn service to interface {} ", interfaceName);
 225         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 226         if (vpnId == VpnConstants.INVALID_ID) {
 227             LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
 228             return;
 229         }
 230         synchronized (interfaceName.intern()) {
 231             if (VpnUtil.getOperationalVpnInterface(broker, vpnInterface.getName()) != null) {
 232                 LOG.trace("VPN Interface already provisioned , bailing out from here.");
 233                 return;
 234             }
 235             bindService(dpId, vpnName, interfaceName, lPortTag);
 236             updateDpnDbs(dpId, vpnName, interfaceName, true);
 237             processVpnInterfaceAdjacencies(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()), vpnInterface);
 238         }
 239
 240     }
 241
 242     private void updateDpnDbs(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
 243         long vpnId = VpnUtil.getVpnId(broker, vpnName);
 244         if (dpId == null) {
 245             dpId = InterfaceUtils.getDpnForInterface(interfaceManager, interfaceName);
 246         }
 247         if(!dpId.equals(BigInteger.ZERO)) {
 248             if(add)
 249                 updateMappingDbs(vpnId, dpId, interfaceName, vpnName);
 250             else
 251                 removeFromMappingDbs(vpnId, dpId, interfaceName, vpnName);
 252         }
 253
 254     }
 255
 256     private void bindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName, int lPortTag) {
 257         int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 258         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 259
 260         int instructionKey = 0;
 261         List<Instruction> instructions = new ArrayList<Instruction>();
 262
 263         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
 264         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
 265
 266         BoundServices
 267             serviceInfo =
 268             InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
 269                                             VpnConstants.L3VPN_SERVICE_IDENTIFIER, priority,
 270                                             VpnConstants.COOKIE_VM_INGRESS_TABLE, instructions);
 271         VpnUtil.syncWrite(broker, LogicalDatastoreType.CONFIGURATION,
 272                           InterfaceUtils.buildServiceId(vpnInterfaceName, VpnConstants.L3VPN_SERVICE_IDENTIFIER), serviceInfo);
 273         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 274                     vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW);
 275         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 276                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.ADD_FLOW);
 277
 278     }
 279
 280     private void processVpnInterfaceAdjacencies(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 281         String intfName = intf.getName();
 282
 283         synchronized (intfName) {
 284             // Read NextHops
 285             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 286             Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
 287
 288             if (adjacencies.isPresent()) {
 289                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 290                 List<Adjacency> value = new ArrayList<>();
 291
 292                 // Get the rd of the vpn instance
 293                 String rd = getRouteDistinguisher(intf.getVpnInstanceName());
 294
 295                 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 296                 if (nextHopIp == null){
 297                     LOG.error("NextHop for interface {} is null", intfName);
 298                 }
 299
 300                 LOG.trace("NextHops are {}", nextHops);
 301                 for (Adjacency nextHop : nextHops) {
 302                     String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 303                     long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, VpnUtil
 304                             .getNextHopLabelKey((rd == null) ? intf.getVpnInstanceName() : rd, prefix));
 305                     String adjNextHop = nextHop.getNextHopIp();
 306                     value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIp((adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp)
 307                             .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 308                     if(nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
 309                         VpnUtil.syncUpdate(
 310                                 broker,
 311                                 LogicalDatastoreType.OPERATIONAL,
 312                                 VpnUtil.getPrefixToInterfaceIdentifier(
 313                                         VpnUtil.getVpnId(broker, intf.getVpnInstanceName()), prefix),
 314                                 VpnUtil.getPrefixToInterface(dpnId, intf.getName(), prefix));
 315                     } else {
 316                         //Extra route adjacency
 317                         VpnUtil.syncUpdate(
 318                                 broker,
 319                                 LogicalDatastoreType.OPERATIONAL,
 320                                 VpnUtil.getVpnToExtrarouteIdentifier(
 321                                         (rd != null) ? rd : intf.getVpnInstanceName(), nextHop.getIpAddress()),
 322                                 VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIp()));
 323
 324                     }
 325                 }
 326
 327                 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 328                 VpnInterface opInterface = VpnUtil.getVpnInterface(intfName, intf.getVpnInstanceName(), aug);
 329                 InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(intfName);
 330                 VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface);
 331                 for (Adjacency nextHop : aug.getAdjacency()) {
 332                     long label = nextHop.getLabel();
 333                     //String adjNextHop = nextHop.getNextHopIp();
 334                     if (rd != null) {
 335                         addPrefixToBGP(rd, nextHop.getIpAddress(),
 336                                             nextHopIp, label);
 337                     } else {
 338                         // ### add FIB route directly
 339                         addFibEntryToDS(intf.getVpnInstanceName(), nextHop.getIpAddress(),
 340                                             nextHopIp, (int) label);
 341                     }
 342                 }
 343             }
 344         }
 345     }
 346
 347     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
 348                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow){
 349         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 350         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, BigInteger.valueOf(vpnId));
 351         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
 352                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
 353
 354         // Matching Arp reply flows
 355         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
 356         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 357                 metadata, metadataMask }));
 358
 359         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 360
 361         // Instruction to punt to controller
 362         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 363         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 364         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
 365         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 366
 367         // Install the flow entry in L3_INTERFACE_TABLE
 368         String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
 369                     NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
 370         FlowEntity flowEntity;
 371         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
 372                 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
 373                 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
 374
 375         if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
 376             LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
 377             mdsalManager.installFlow(flowEntity);
 378         } else {
 379             LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
 380             mdsalManager.removeFlow(flowEntity);
 381         }
 382     }
 383
 384     private String getRouteDistinguisher(String vpnName) {
 385         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 386                                       .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 387         Optional<VpnInstance> vpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 388         String rd = "";
 389         if(vpnInstance.isPresent()) {
 390             VpnInstance instance = vpnInstance.get();
 391             VpnAfConfig config = instance.getIpv4Family();
 392             rd = config.getRouteDistinguisher();
 393         }
 394         return rd;
 395     }
 396
 397     private synchronized void updateMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 398         String routeDistinguisher = getRouteDistinguisher(vpnName);
 399         String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
 400         InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 401         Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 402         org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 403             vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 404
 405         if (dpnInVpn.isPresent()) {
 406             VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 407                 org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance
 408                     .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 409                     new VpnInterfacesKey(intfName)), vpnInterface);
 410         } else {
 411             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 412                                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 413                                     VpnUtil.getVpnInstanceOpDataBuilder(rd, vpnId));
 414             VpnToDpnListBuilder vpnToDpnList = new VpnToDpnListBuilder().setDpnId(dpnId);
 415             List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 416                 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces =  new ArrayList<>();
 417             vpnInterfaces.add(vpnInterface);
 418             VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id,
 419                               vpnToDpnList.setVpnInterfaces(vpnInterfaces).build());
 420
 421             /**
 422              * FIXME: DC Gateway tunnel should be built dynamically
 423             //this is the first VM in this VPN on the DPN, may be a new DPN has come up,
 424             //if tunnel to DC GW does not exist, create it
 425             //if(!tunnelExists(dpnID, bgpManager.getDCGWIP()))
 426             String dcGW = bgpManager.getDCGwIP();
 427             if(dcGW != null && !dcGW.isEmpty())
 428             {
 429                 LOG.debug("Building tunnel from DPN {} to DC GW {}", dpnId, dcGW);
 430                 itmProvider.buildTunnelFromDPNToDCGW(dpnId, new IpAddress(dcGW.toCharArray()));
 431             }*/
 432             fibManager.populateFibOnNewDpn(dpnId, vpnId, (rd == null) ? vpnName : rd);
 433         }
 434     }
 435
 436     private synchronized void removeFromMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
 437         //TODO: Delay 'DPN' removal so that other services can cleanup the entries for this dpn
 438         String rd = VpnUtil.getVpnRd(broker, vpnName);
 439         InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 440         Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 441         if (dpnInVpn.isPresent()) {
 442             List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 443                 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 444             org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
 445                     currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 446
 447             if (vpnInterfaces.remove(currVpnInterface)) {
 448                 if (vpnInterfaces.isEmpty()) {
 449                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id, VpnUtil.DEFAULT_CALLBACK);
 450                     fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
 451                 } else {
 452                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
 453                         org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 454                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
 455                             new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
 456                 }
 457             }
 458         }
 459     }
 460
 461     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label) {
 462         try {
 463             bgpManager.addPrefix(rd, prefix, nextHopIp, (int)label);
 464         } catch(Exception e) {
 465             LOG.error("Add prefix failed", e);
 466         }
 467     }
 468
 469
 470     private InstanceIdentifier<VpnInterface> getWildCardPath() {
 471         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 472     }
 473
 474     @Override
 475     protected void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
 476         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
 477         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 478         String interfaceName = key.getName();
 479
 480         InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 481         Optional<VpnInterface> existingVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, interfaceId);
 482         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
 483             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
 484
 485         if (existingVpnInterface.isPresent() && interfaceState != null) {
 486             processVpnInterfaceDown(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName, interfaceState.getIfIndex(), false);
 487         } else {
 488             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
 489         }
 490     }
 491
 492     protected void processVpnInterfaceDown(BigInteger dpId, String interfaceName, int lPortTag, boolean isInterfaceStateDown) {
 493         VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
 494         if(vpnInterface == null) {
 495             LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
 496             return;
 497         }
 498         String vpnName = vpnInterface.getVpnInstanceName();
 499         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 500
 501         synchronized (interfaceName.intern()) {
 502             removeAdjacenciesFromVpn(identifier, vpnInterface);
 503             LOG.info("Unbinding vpn service from interface {} ", interfaceName);
 504             unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown);
 505
 506             //wait till DCN for removal of vpn interface in operational DS arrives
 507             Runnable notifyTask = new VpnNotifyTask();
 508             synchronized (interfaceName.intern()) {
 509                 vpnIntfMap.put(interfaceName, notifyTask);
 510                 synchronized (notifyTask) {
 511                     try {
 512                         notifyTask.wait(VpnConstants.MIN_WAIT_TIME_IN_MILLISECONDS);
 513                     } catch (InterruptedException e) {
 514                     }
 515                 }
 516             }
 517
 518         }
 519     }
 520
 521     private void removeAdjacenciesFromVpn(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 522         //Read NextHops
 523         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 524         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 525
 526         String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
 527         LOG.trace("removeAdjacenciesFromVpn: For interface {} RD recovered for vpn {} as rd {}", intf.getName(),
 528                 intf.getVpnInstanceName(), rd);
 529         if (adjacencies.isPresent()) {
 530             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 531
 532             if (!nextHops.isEmpty()) {
 533                 LOG.trace("NextHops are " + nextHops);
 534                 for (Adjacency nextHop : nextHops) {
 535                     // Commenting the release of ID here as it will be released by FIB
 536                    /* VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 537                                       VpnUtil.getNextHopLabelKey(rd, nextHop.getIpAddress()));
 538                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
 539                                    VpnUtil.getPrefixToInterfaceIdentifier(
 540                                        VpnUtil.getVpnId(broker, intf.getVpnInstanceName()),
 541                                        nextHop.getIpAddress()),
 542                                    VpnUtil.DEFAULT_CALLBACK);*/
 543                     if (rd.equals(intf.getVpnInstanceName())) {
 544                         //this is an internal vpn - the rd is assigned to the vpn instance name;
 545                         //remove from FIB directly
 546                         removeFibEntryFromDS(intf.getVpnInstanceName(), nextHop.getIpAddress());
 547                     } else {
 548                         removePrefixFromBGP(rd, nextHop.getIpAddress());
 549                     }
 550                 }
 551             }
 552         }
 553     }
 554
 555
 556     private void unbindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName,
 557                                int lPortTag, boolean isInterfaceStateDown) {
 558         if (!isInterfaceStateDown) {
 559             VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION,
 560                            InterfaceUtils.buildServiceId(vpnInterfaceName,
 561                                                          VpnConstants.L3VPN_SERVICE_IDENTIFIER),
 562                            VpnUtil.DEFAULT_CALLBACK);
 563         }
 564         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 565         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 566                     vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW);
 567         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
 568                 vpnId, ArpReplyOrRequest.REPLY, NwConstants.DEL_FLOW);
 569     }
 570
 571
 572     private void removePrefixFromBGP(String rd, String prefix) {
 573         try {
 574             bgpManager.deletePrefix(rd, prefix);
 575         } catch(Exception e) {
 576             LOG.error("Delete prefix failed", e);
 577         }
 578     }
 579
 580     @Override
 581     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
 582         if (LOG.isTraceEnabled()) {
 583             LOG.trace("Updating VPN Interface : key " + identifier + ",  original value=" + original + ", update " +
 584                     "value=" + update);
 585         }
 586         String oldVpnName = original.getVpnInstanceName();
 587         String newVpnName = update.getVpnInstanceName();
 588         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
 589         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
 590         if (oldAdjs == null) {
 591             oldAdjs = new ArrayList<>();
 592         }
 593         if (newAdjs == null) {
 594             newAdjs = new ArrayList<>();
 595         }
 596         //handles switching between <internal VPN - external VPN>
 597         if (!oldVpnName.equals(newVpnName)) {
 598             remove(identifier, original);
 599             add(identifier, update);
 600         }
 601         //handle both addition and removal of adjacencies
 602         //currently, new adjacency may be an extra route
 603         if (!oldAdjs.equals(newAdjs)) {
 604             for (Adjacency adj : newAdjs) {
 605                 if (oldAdjs.contains(adj)) {
 606                     oldAdjs.remove(adj);
 607                 } else {
 608                     // add new adjacency - right now only extra route will hit this path
 609                     addNewAdjToVpnInterface(identifier, adj);
 610                 }
 611             }
 612             for (Adjacency adj : oldAdjs) {
 613                 delAdjFromVpnInterface(identifier, adj);
 614             }
 615         }
 616     }
 617
 618     public void processArpRequest(IpAddress srcIP, PhysAddress srcMac, IpAddress targetIP, String srcInterface){
 619         SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
 620                                                                     .setIpaddress(srcIP).setSrcIpAddress(targetIP).setMacaddress(srcMac).build();
 621         final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
 622         Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
 623         Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
 624             @Override
 625             public void onFailure(Throwable error) {
 626                 LOG.error("Error - {}", msgFormat, error);
 627             }
 628
 629             @Override
 630             public void onSuccess(RpcResult<Void> result) {
 631                 if(!result.isSuccessful()) {
 632                     LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
 633                 } else {
 634                     LOG.debug("Successful RPC Result - {}", msgFormat);
 635                 }
 636             }
 637         });
 638     }
 639
 640     private String getErrorText(Collection<RpcError> errors) {
 641         StringBuilder errorText = new StringBuilder();
 642         for(RpcError error : errors) {
 643             errorText.append(",").append(error.getErrorType()).append("-")
 644                      .append(error.getMessage());
 645         }
 646         return errorText.toString();
 647     }
 648
 649     private String getTunnelInterfaceFlowRef(BigInteger dpnId, short tableId, String ifName) {
 650         return new StringBuilder().append(dpnId).append(tableId).append(ifName).toString();
 651     }
 652
 653
 654
 655     public synchronized void addFibEntryToDS(String rd, String prefix,
 656             String nexthop, int label) {
 657
 658         VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).
 659                     setNextHopAddress(nexthop).setLabel((long)label).build();
 660         LOG.debug("Created vrfEntry for {} nexthop {} label {}", prefix, nexthop, label);
 661
 662         List<VrfEntry> vrfEntryList = new ArrayList<VrfEntry>();
 663         vrfEntryList.add(vrfEntry);
 664
 665         InstanceIdentifierBuilder<VrfTables> idBuilder =
 666                     InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
 667         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
 668
 669         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
 670                     setVrfEntry(vrfEntryList).build();
 671
 672         VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
 673     }
 674
 675     public synchronized void removeFibEntryFromDS(String rd, String prefix) {
 676
 677         LOG.debug("Removing fib entry with destination prefix {} from vrf table for rd {}", prefix, rd);
 678
 679         InstanceIdentifierBuilder<VrfEntry> idBuilder =
 680             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).child(VrfEntry.class, new VrfEntryKey(prefix));
 681         InstanceIdentifier<VrfEntry> vrfEntryId = idBuilder.build();
 682         VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId, VpnUtil.DEFAULT_CALLBACK);
 683
 684     }
 685
 686     public synchronized void removeVrfFromDS(String rd) {
 687         LOG.debug("Removing vrf table for  rd {}", rd);
 688
 689         InstanceIdentifierBuilder<VrfTables> idBuilder =
 690                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
 691         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
 692
 693         VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, VpnUtil.DEFAULT_CALLBACK);
 694
 695     }
 696
 697     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
 698
 699         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
 700
 701         if (optVpnInterface.isPresent()) {
 702             VpnInterface currVpnIntf = optVpnInterface.get();
 703             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
 704             String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
 705             InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
 706             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, adjPath);
 707             long label =
 708                     VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 709                             VpnUtil.getNextHopLabelKey((rd != null) ? rd : currVpnIntf.getVpnInstanceName(), prefix));
 710
 711             List<Adjacency> adjacencies;
 712             if (optAdjacencies.isPresent()) {
 713                 adjacencies = optAdjacencies.get().getAdjacency();
 714             } else {
 715                 //This code will not be hit since VM adjacency will always be there
 716                 adjacencies = new ArrayList<>();
 717             }
 718
 719             adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIp(adj.getNextHopIp())
 720                     .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 721
 722             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
 723             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug);
 724
 725             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
 726             addExtraRoute(adj.getIpAddress(), adj.getNextHopIp(), rd, currVpnIntf.getVpnInstanceName(), (int) label, currVpnIntf.getName());
 727
 728         }
 729
 730     }
 731
 732     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
 733         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
 734
 735         if (optVpnInterface.isPresent()) {
 736             VpnInterface currVpnIntf = optVpnInterface.get();
 737
 738             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 739             Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 740             if (optAdjacencies.isPresent()) {
 741                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 742
 743                 if (!adjacencies.isEmpty()) {
 744                     String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
 745                     LOG.trace("Adjacencies are " + adjacencies);
 746                     Iterator<Adjacency> adjIt = adjacencies.iterator();
 747                     while (adjIt.hasNext()) {
 748                         Adjacency adjElem = adjIt.next();
 749                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
 750                             // Commenting the release of ID here as it will be released by FIB
 751                            /* VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 752                                     VpnUtil.getNextHopLabelKey(rd, adj.getIpAddress()));*/
 753                             adjIt.remove();
 754
 755                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
 756                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
 757                                                                               currVpnIntf.getVpnInstanceName(),
 758                                                                               aug);
 759
 760                             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
 761
 762                             delExtraRoute(adj.getIpAddress(), rd, currVpnIntf.getVpnInstanceName());
 763                             break;
 764                         }
 765
 766                     }
 767                 }
 768             }
 769         }
 770
 771     }
 772
 773     protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label, String intfName) {
 774
 775         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 776         VpnUtil.syncUpdate(
 777             broker,
 778             LogicalDatastoreType.OPERATIONAL,
 779             VpnUtil.getVpnToExtrarouteIdentifier(
 780                 (rd != null) ? rd : routerID, destination),
 781             VpnUtil.getVpnToExtraroute(destination, nextHop));
 782
 783         if (intfName != null && !intfName.isEmpty()) {
 784             BigInteger dpnId = InterfaceUtils.getDpnForInterface(interfaceManager, intfName);
 785             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
 786             if (nextHopIp == null || nextHopIp.isEmpty()) {
 787                 LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}", intfName, destination);
 788             }
 789             nextHop = nextHopIp;
 790         }
 791         if (rd != null) {
 792             addPrefixToBGP(rd, destination, nextHop, label);
 793         } else {
 794             // ### add FIB route directly
 795             addFibEntryToDS(routerID, destination, nextHop, label);
 796         }
 797     }
 798
 799     protected void delExtraRoute(String destination, String rd, String routerID) {
 800         if (rd != null) {
 801             removePrefixFromBGP(rd, destination);
 802         } else {
 803             // ### add FIB route directly
 804             removeFibEntryFromDS(routerID, destination);
 805         }
 806     }
 807
 808     class VpnInterfaceOpListener extends org.opendaylight.vpnservice.mdsalutil.AbstractDataChangeListener<VpnInterface> {
 809
 810         public VpnInterfaceOpListener() {
 811             super(VpnInterface.class);
 812         }
 813
 814         @Override
 815         protected void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface del) {
 816             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 817             String interfaceName = key.getName();
 818             String vpnName = del.getVpnInstanceName();
 819
 820             LOG.trace("VpnInterfaceOpListener removed: interface name {} vpnName {}", interfaceName, vpnName);
 821             //decrement the vpn interface count in Vpn Instance Op Data
 822             InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance>
 823                     id = VpnUtil.getVpnInstanceToVpnIdIdentifier(vpnName);
 824             Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance> vpnInstance
 825                     = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
 826
 827             if (vpnInstance.isPresent()) {
 828                 String rd = null;
 829                 rd = vpnInstance.get().getVrfId();
 830                 //String rd = getRouteDistinguisher(del.getVpnInstanceName());
 831
 832                 VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
 833                 LOG.trace("VpnInterfaceOpListener removed: interface name {} rd {} vpnName {} in Vpn Op Instance {}",
 834                         interfaceName, rd, vpnName, vpnInstOp);
 835
 836                 if (vpnInstOp != null) {
 837                     // Vpn Interface removed => No more adjacencies from it.
 838                     // Hence clean up interface from vpn-dpn-interface list.
 839                     Adjacency adjacency = del.getAugmentation(Adjacencies.class).getAdjacency().get(0);
 840                     Optional<Prefixes> prefixToInterface = Optional.absent();
 841                     prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 842                             VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
 843                                     VpnUtil.getIpPrefix(adjacency.getIpAddress())));
 844                     if (!prefixToInterface.isPresent()) {
 845                         prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 846                                 VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
 847                                         VpnUtil.getIpPrefix(adjacency.getNextHopIp())));
 848                     }
 849                     if (prefixToInterface.isPresent()) {
 850                         VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
 851                                 VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
 852                                         prefixToInterface.get().getIpAddress()),
 853                                 VpnUtil.DEFAULT_CALLBACK);
 854                         updateDpnDbs(prefixToInterface.get().getDpnId(), del.getVpnInstanceName(), interfaceName, false);
 855                     }
 856                     Long ifCnt = 0L;
 857                     ifCnt = vpnInstOp.getVpnInterfaceCount();
 858                     LOG.trace("VpnInterfaceOpListener removed: interface name {} rd {} vpnName {} Intf count {}",
 859                             interfaceName, rd, vpnName, ifCnt);
 860                     if ((ifCnt != null) && (ifCnt > 0)) {
 861                         VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 862                                 VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 863                                 VpnUtil.updateIntfCntInVpnInstOpData(ifCnt - 1, rd), VpnUtil.DEFAULT_CALLBACK);
 864                     }
 865                 }
 866             } else {
 867                 LOG.error("rd not retrievable as vpninstancetovpnid for vpn {} is absent, trying rd as ", vpnName, vpnName);
 868             }
 869             notifyTaskIfRequired(interfaceName);
 870         }
 871
 872         private void notifyTaskIfRequired(String intfName) {
 873             Runnable notifyTask = vpnIntfMap.remove(intfName);
 874             if (notifyTask == null) {
 875                 return;
 876             }
 877             executorService.execute(notifyTask);
 878         }
 879
 880         @Override
 881         protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
 882         }
 883
 884         @Override
 885         protected void add(InstanceIdentifier<VpnInterface> identifier, VpnInterface add) {
 886             final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 887             String interfaceName = key.getName();
 888
 889             //increment the vpn interface count in Vpn Instance Op Data
 890             Long ifCnt = 0L;
 891             String rd = getRouteDistinguisher(add.getVpnInstanceName());
 892             if(rd == null || rd.isEmpty()) rd = add.getVpnInstanceName();
 893             VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
 894             if(vpnInstOp != null &&  vpnInstOp.getVpnInterfaceCount() != null) {
 895                 ifCnt = vpnInstOp.getVpnInterfaceCount();
 896             }
 897
 898             LOG.trace("VpnInterfaceOpListener add: interface name {} rd {} interface count in Vpn Op Instance {}", interfaceName, rd, ifCnt);
 899
 900             VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 901                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 902                     VpnUtil.updateIntfCntInVpnInstOpData(ifCnt + 1, rd), VpnUtil.DEFAULT_CALLBACK);
 903
 904
 905         }
 906     }
 907
 908     protected void updatePrefixesForDPN(BigInteger dpnId, UpdateRouteAction action) {
 909
 910         LOG.info("Tunnel event triggered {} for Dpn:{} ", action.name(), dpnId);
 911         InstanceIdentifierBuilder<VpnInstances> idBuilder = InstanceIdentifier.builder(VpnInstances.class);
 912         InstanceIdentifier<VpnInstances> vpnInstancesId = idBuilder.build();
 913         Optional<VpnInstances> vpnInstances = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnInstancesId);
 914
 915         if (vpnInstances.isPresent()) {
 916             List<VpnInstance> vpnInstanceList = vpnInstances.get().getVpnInstance();
 917             Iterator<VpnInstance> vpnInstIter = vpnInstanceList.iterator();
 918             while (vpnInstIter.hasNext()) {
 919                 VpnInstance vpnInstance = vpnInstIter.next();
 920                 try {
 921                     VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
 922                     String rd = vpnConfig.getRouteDistinguisher();
 923                     if (rd == null || rd.isEmpty()) {
 924                         rd = vpnInstance.getVpnInstanceName();
 925                     }
 926                     InstanceIdentifier<VpnToDpnList> id =
 927                         VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
 928                     Optional<VpnToDpnList> dpnInVpn =
 929                         VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 930                     if (dpnInVpn.isPresent()) {
 931                         // if (action == UpdateRouteAction.ADVERTISE_ROUTE) {
 932                         //    fibManager.populateFibOnNewDpn(dpnId, VpnUtil
 933                         //        .getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
 934                         // }
 935                         List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 936                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces>
 937                             vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
 938                         for (org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
 939                             .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces vpnInterface : vpnInterfaces) {
 940                             InstanceIdentifier<VpnInterface> vpnIntfId =
 941                                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getInterfaceName());
 942                             InstanceIdentifier<Adjacencies> path =
 943                                 vpnIntfId.augmentation(Adjacencies.class);
 944                             Optional<Adjacencies> adjacencies =
 945                                 VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 946
 947                             if (adjacencies.isPresent()) {
 948                                 List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
 949                                 Iterator<Adjacency> adjacencyIterator = adjacencyList.iterator();
 950
 951                                 while (adjacencyIterator.hasNext()) {
 952                                     Adjacency adjacency = adjacencyIterator.next();
 953                                     try {
 954                                         if (action == UpdateRouteAction.ADVERTISE_ROUTE)
 955                                             bgpManager.addPrefix(rd, adjacency.getIpAddress(),
 956                                                                  adjacency.getNextHopIp(),
 957                                                                  adjacency.getLabel().intValue());
 958                                         else if (action == UpdateRouteAction.WITHDRAW_ROUTE)
 959                                             bgpManager.deletePrefix(rd, adjacency.getIpAddress());
 960                                     } catch (Exception e) {
 961                                         LOG.error("Exception when updating prefix {} in vrf {} to BGP",
 962                                             adjacency.getIpAddress(), rd);
 963                                     }
 964                                 }
 965                             }
 966
 967                         }
 968                         // if (action == UpdateRouteAction.WITHDRAW_ROUTE) {
 969                         //    fibManager.cleanUpDpnForVpn(dpnId, VpnUtil.getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
 970                         // }
 971                     }
 972                 } catch (Exception e) {
 973                     LOG.error("updatePrefixesForDPN {} in vpn {} failed", dpnId, vpnInstance.getVpnInstanceName(), e);
 974                 }
 975             }
 976         }
 977     }
 978
 979         InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
 980         return InstanceIdentifier.builder(NeutronRouterDpns.class)
 981             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
 982             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
 983     }
 984
 985     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
 986         return InstanceIdentifier.builder(NeutronRouterDpns.class)
 987             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
 988     }
 989
 990     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName) {
 991         BigInteger dpId = InterfaceUtils.getDpnForInterface(interfaceManager, vpnInterfaceName);
 992         if(dpId.equals(BigInteger.ZERO)) {
 993             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
 994             return;
 995         }
 996         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
 997
 998         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
 999                 .CONFIGURATION, routerDpnListIdentifier);
1000         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1001         if (optionalRouterDpnList.isPresent()) {
1002             MDSALUtil.syncWrite(broker, LogicalDatastoreType.CONFIGURATION, routerDpnListIdentifier.child(
1003                     RouterInterfaces.class,  new RouterInterfacesKey(vpnInterfaceName)), routerInterface);
1004         } else {
1005             MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION,
1006                     getRouterId(routerName),
1007                     new RouterDpnListBuilder().setRouterId(routerName).build());
1008             //VpnToDpnListBuilder vpnToDpnList = new VpnToDpnListBuilder().setDpnId(dpnId);
1009             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1010             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
1011             routerInterfaces.add(routerInterface);
1012             MDSALUtil.syncWrite(broker, LogicalDatastoreType.CONFIGURATION, routerDpnListIdentifier,
1013                     dpnVpnList.setRouterInterfaces(routerInterfaces).build());
1014         }
1015     }
1016
1017     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName) {
1018         BigInteger dpId = InterfaceUtils.getDpnForInterface(interfaceManager, vpnInterfaceName);
1019         if(dpId.equals(BigInteger.ZERO)) {
1020             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1021             return;
1022         }
1023         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1024         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
1025                 .CONFIGURATION, routerDpnListIdentifier);
1026         if (optionalRouterDpnList.isPresent()) {
1027             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1028             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1029
1030             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1031                 if (routerInterfaces.isEmpty()) {
1032                     MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, routerDpnListIdentifier);
1033                 } else {
1034                     MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, routerDpnListIdentifier.child(
1035                             RouterInterfaces.class,
1036                             new RouterInterfacesKey(vpnInterfaceName)));
1037                 }
1038             }
1039         }
1040     }
1041
1042         protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,BigInteger dpId) {
1043         if(dpId.equals(BigInteger.ZERO)) {
1044             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
1045             return;
1046         }
1047         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1048         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(broker, LogicalDatastoreType
1049                 .CONFIGURATION, routerDpnListIdentifier);
1050         if (optionalRouterDpnList.isPresent()) {
1051             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
1052             RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
1053
1054             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
1055                 if (routerInterfaces.isEmpty()) {
1056                     MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, routerDpnListIdentifier);
1057                 } else {
1058                     MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, routerDpnListIdentifier.child(
1059                             RouterInterfaces.class,
1060                             new RouterInterfacesKey(vpnInterfaceName)));
1061                 }
1062             }
1063         }
1064     }
1065
1066 }