2 * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.vpnservice;
10 import org.opendaylight.vpnservice.utilities.InterfaceUtils;
12 import com.google.common.util.concurrent.FutureCallback;
13 import com.google.common.util.concurrent.Futures;
14 import com.google.common.util.concurrent.JdkFutureAdapters;
16 import org.opendaylight.controller.md.sal.binding.api.*;
17 import org.opendaylight.vpnservice.mdsalutil.*;
18 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
19 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
20 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyKey;
21 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
22 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
23 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
24 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
25 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
26 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
27 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.OdlArputilService;
28 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.SendArpResponseInput;
29 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.arputil.rev151126.SendArpResponseInputBuilder;
30 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.servicebinding.rev151015.service.bindings.services.info.BoundServices;
31 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.FibEntries;
32 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTables;
33 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesBuilder;
34 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesKey;
35 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntry;
36 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
37 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.vrfentries.VrfEntryKey;
38 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.IdManagerService;
40 import java.math.BigInteger;
41 import java.util.Collection;
42 import java.util.Iterator;
43 import java.util.List;
44 import java.util.ArrayList;
45 import java.util.concurrent.*;
47 import com.google.common.base.Optional;
49 import org.opendaylight.bgpmanager.api.IBgpManager;
50 import org.opendaylight.fibmanager.api.IFibManager;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.interfacemgr.rpcs.rev151003.OdlInterfaceRpcService;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.itm.rpcs.rev151217.ItmRpcService;
53 import org.opendaylight.vpnservice.mdsalutil.interfaces.IMdsalApiManager;
54 import org.opendaylight.yangtools.concepts.ListenerRegistration;
55 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
56 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
57 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
58 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.Adjacency;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
63 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.Adjacencies;
64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
69 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
70 import org.opendaylight.yangtools.yang.common.RpcError;
71 import org.opendaylight.yangtools.yang.common.RpcResult;
72 import org.slf4j.Logger;
73 import org.slf4j.LoggerFactory;
75 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
76 private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
77 private ListenerRegistration<DataChangeListener> listenerRegistration, opListenerRegistration;
78 private ConcurrentMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
79 private ExecutorService executorService = Executors.newSingleThreadExecutor();
80 private final DataBroker broker;
81 private final IBgpManager bgpManager;
82 private IFibManager fibManager;
83 private IMdsalApiManager mdsalManager;
84 private OdlInterfaceRpcService interfaceManager;
85 private ItmRpcService itmProvider;
86 private IdManagerService idManager;
87 private OdlArputilService arpManager;
88 private InterfaceStateChangeListener interfaceListener;
89 private VpnInterfaceOpListener vpnInterfaceOpListener;
90 private ArpNotificationHandler arpNotificationHandler;
91 protected enum UpdateRouteAction {
92 ADVERTISE_ROUTE, WITHDRAW_ROUTE
95 * Responsible for listening to data change related to VPN Interface
96 * Bind VPN Service on the interface and informs the BGP service
98 * @param db - dataBroker service reference
100 public VpnInterfaceManager(final DataBroker db, final IBgpManager bgpManager, NotificationService notificationService) {
101 super(VpnInterface.class);
103 this.bgpManager = bgpManager;
104 interfaceListener = new InterfaceStateChangeListener(db, this);
105 vpnInterfaceOpListener = new VpnInterfaceOpListener();
106 arpNotificationHandler = new ArpNotificationHandler(this, broker);
107 notificationService.registerNotificationListener(arpNotificationHandler);
108 registerListener(db);
111 public void setMdsalManager(IMdsalApiManager mdsalManager) {
112 this.mdsalManager = mdsalManager;
115 public void setInterfaceManager(OdlInterfaceRpcService interfaceManager) {
116 this.interfaceManager = interfaceManager;
117 interfaceListener.setInterfaceManager(interfaceManager);
120 public void setITMProvider(ItmRpcService itmProvider) {
121 this.itmProvider = itmProvider;
124 public void setFibManager(IFibManager fibManager) {
125 this.fibManager = fibManager;
128 public void setIdManager(IdManagerService idManager) {
129 this.idManager = idManager;
132 public void setArpManager(OdlArputilService arpManager) {
133 this.arpManager = arpManager;
137 public void close() throws Exception {
138 if (listenerRegistration != null) {
140 listenerRegistration.close();
141 opListenerRegistration.close();
142 } catch (final Exception e) {
143 LOG.error("Error when cleaning up DataChangeListener.", e);
145 listenerRegistration = null;
146 opListenerRegistration = null;
148 LOG.info("VPN Interface Manager Closed");
151 private void registerListener(final DataBroker db) {
153 listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
154 getWildCardPath(), VpnInterfaceManager.this, DataChangeScope.SUBTREE);
155 opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
156 getWildCardPath(), vpnInterfaceOpListener, DataChangeScope.SUBTREE);
157 } catch (final Exception e) {
158 LOG.error("VPN Service DataChange listener registration fail!", e);
159 throw new IllegalStateException("VPN Service registration Listener failed.", e);
163 private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
164 return InstanceIdentifier.create(InterfacesState.class)
165 .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
169 protected void add(final InstanceIdentifier<VpnInterface> identifier,
170 final VpnInterface vpnInterface) {
171 LOG.trace("VPN Interface key: {} , value: {}", identifier, vpnInterface );
172 addInterface(identifier, vpnInterface);
175 private void addInterface(final InstanceIdentifier<VpnInterface> identifier,
176 final VpnInterface vpnInterface) {
177 LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
178 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
179 String interfaceName = key.getName();
181 org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
182 InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
183 if (interfaceState != null) {
184 // Interface state is up
185 processVpnInterfaceUp(interfaceName, interfaceState.getIfIndex());
187 LOG.trace("VPN interfaces are not yet operational.");
191 protected void processVpnInterfaceUp(String interfaceName, int lPortTag) {
193 VpnInterface vpnInterface = VpnUtil.getConfiguredVpnInterface(broker, interfaceName);
194 if(vpnInterface == null) {
195 LOG.info("Unable to process add/up for interface {} as it is not configured", interfaceName);
198 String vpnName = vpnInterface.getVpnInstanceName();
199 LOG.info("Binding vpn service to interface {} ", interfaceName);
200 long vpnId = VpnUtil.getVpnId(broker, vpnName);
201 if (vpnId == VpnConstants.INVALID_ID) {
202 LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
205 if (VpnUtil.getOperationalVpnInterface(broker, vpnInterface.getName()) != null) {
206 LOG.trace("VPN Interface already provisioned , bailing out from here.");
209 synchronized (interfaceName.intern()) {
211 bindService(vpnName, interfaceName, lPortTag);
212 updateDpnDbs(vpnName, interfaceName, true);
213 processVpnInterfaceAdjacencies(VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()), vpnInterface);
218 private void updateDpnDbs(String vpnName, String interfaceName, boolean add) {
219 long vpnId = VpnUtil.getVpnId(broker, vpnName);
220 BigInteger dpId = InterfaceUtils.getDpnForInterface(interfaceManager, interfaceName);
221 if(!dpId.equals(BigInteger.ZERO)) {
223 updateMappingDbs(vpnId, dpId, interfaceName, vpnName);
225 removeFromMappingDbs(vpnId, dpId, interfaceName, vpnName);
230 private void bindService(String vpnInstanceName, String vpnInterfaceName, int lPortTag) {
231 int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
232 long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
234 int instructionKey = 0;
235 List<Instruction> instructions = new ArrayList<Instruction>();
237 instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
238 instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
242 InterfaceUtils.getBoundServices(String.format("%s.%s.%s", "vpn",vpnInstanceName, vpnInterfaceName),
243 VpnConstants.L3VPN_SERVICE_IDENTIFIER, priority,
244 VpnConstants.COOKIE_VM_INGRESS_TABLE, instructions);
245 VpnUtil.syncWrite(broker, LogicalDatastoreType.CONFIGURATION,
246 InterfaceUtils.buildServiceId(vpnInterfaceName, VpnConstants.L3VPN_SERVICE_IDENTIFIER), serviceInfo);
247 makeArpFlow(VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName, vpnId, ArpReplyOrRequest.REQUEST, NwConstants.ADD_FLOW);
251 private void processVpnInterfaceAdjacencies(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
252 String intfName = intf.getName();
254 synchronized (intfName) {
256 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
257 Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
259 if (adjacencies.isPresent()) {
260 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
261 List<Adjacency> value = new ArrayList<>();
263 // Get the rd of the vpn instance
264 String rd = getRouteDistinguisher(intf.getVpnInstanceName());
266 BigInteger dpnId = InterfaceUtils.getDpnForInterface(interfaceManager, intfName);
267 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
268 if (nextHopIp == null){
269 LOG.error("NextHop for interface {} is null", intfName);
272 LOG.trace("NextHops are {}", nextHops);
273 for (Adjacency nextHop : nextHops) {
274 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
275 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, VpnUtil
276 .getNextHopLabelKey((rd == null) ? intf.getVpnInstanceName() : rd, prefix));
277 String adjNextHop = nextHop.getNextHopIp();
278 value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIp((adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp)
279 .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
280 if(nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
283 LogicalDatastoreType.OPERATIONAL,
284 VpnUtil.getPrefixToInterfaceIdentifier(
285 VpnUtil.getVpnId(broker, intf.getVpnInstanceName()), prefix),
286 VpnUtil.getPrefixToInterface(dpnId, intf.getName(), prefix));
290 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
291 VpnInterface opInterface = VpnUtil.getVpnInterface(intfName, intf.getVpnInstanceName(), aug);
292 InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(intfName);
293 VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface);
294 for (Adjacency nextHop : aug.getAdjacency()) {
295 long label = nextHop.getLabel();
296 String adjNextHop = nextHop.getNextHopIp();
298 addPrefixToBGP(rd, nextHop.getIpAddress(),
299 (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp, label);
301 // ### add FIB route directly
302 addFibEntryToDS(intf.getVpnInstanceName(), nextHop.getIpAddress(),
303 (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp, (int) label);
310 private void makeArpFlow(short sIndex, int lPortTag, String vpnInterfaceName, long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow){
311 List<MatchInfo> matches = new ArrayList<MatchInfo>();
312 BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, BigInteger.valueOf(vpnId));
313 BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
314 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
316 // Matching Arp reply flows
317 matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_ARP }));
318 matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
319 metadata, metadataMask }));
321 matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
323 // Instruction to punt to controller
324 List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
325 List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
326 actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
327 instructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
329 // Install the flow entry in L3_INTERFACE_TABLE
330 BigInteger dpId = InterfaceUtils.getDpnForInterface(interfaceManager, vpnInterfaceName);
331 String flowRef = VpnUtil.getFlowRef(dpId, NwConstants.L3_INTERFACE_TABLE,
332 NwConstants.ETHTYPE_ARP, lPortTag, replyOrRequest.getArpOperation());
333 FlowEntity flowEntity;
334 flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_INTERFACE_TABLE, flowRef,
335 NwConstants.DEFAULT_ARP_FLOW_PRIORITY, replyOrRequest.getName(), 0, 0,
336 VpnUtil.getCookieArpFlow(lPortTag), matches, instructions);
338 if (addOrRemoveFlow == NwConstants.ADD_FLOW) {
339 LOG.debug("Creating ARP Flow for interface {}",vpnInterfaceName);
340 mdsalManager.installFlow(flowEntity);
342 LOG.debug("Deleting ARP Flow for interface {}",vpnInterfaceName);
343 mdsalManager.removeFlow(flowEntity);
347 private String getRouteDistinguisher(String vpnName) {
348 InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
349 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
350 Optional<VpnInstance> vpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
352 if(vpnInstance.isPresent()) {
353 VpnInstance instance = vpnInstance.get();
354 VpnAfConfig config = instance.getIpv4Family();
355 rd = config.getRouteDistinguisher();
360 private synchronized void updateMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
361 String routeDistinguisher = getRouteDistinguisher(vpnName);
362 String rd = (routeDistinguisher == null) ? vpnName : routeDistinguisher;
363 InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
364 Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
365 org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
366 vpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
368 if (dpnInVpn.isPresent()) {
369 VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id.child(
370 org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance
371 .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
372 new VpnInterfacesKey(intfName)), vpnInterface);
374 VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
375 VpnUtil.getVpnInstanceOpDataIdentifier(rd),
376 VpnUtil.getVpnInstanceOpDataBuilder(rd, vpnId));
377 VpnToDpnListBuilder vpnToDpnList = new VpnToDpnListBuilder().setDpnId(dpnId);
378 List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
379 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = new ArrayList<>();
380 vpnInterfaces.add(vpnInterface);
381 VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id,
382 vpnToDpnList.setVpnInterfaces(vpnInterfaces).build());
385 * FIXME: DC Gateway tunnel should be built dynamically
386 //this is the first VM in this VPN on the DPN, may be a new DPN has come up,
387 //if tunnel to DC GW does not exist, create it
388 //if(!tunnelExists(dpnID, bgpManager.getDCGWIP()))
389 String dcGW = bgpManager.getDCGwIP();
390 if(dcGW != null && !dcGW.isEmpty())
392 LOG.debug("Building tunnel from DPN {} to DC GW {}", dpnId, dcGW);
393 itmProvider.buildTunnelFromDPNToDCGW(dpnId, new IpAddress(dcGW.toCharArray()));
395 fibManager.populateFibOnNewDpn(dpnId, vpnId, (rd == null) ? vpnName : rd);
399 private synchronized void removeFromMappingDbs(long vpnId, BigInteger dpnId, String intfName, String vpnName) {
400 //TODO: Delay 'DPN' removal so that other services can cleanup the entries for this dpn
401 String rd = VpnUtil.getVpnRd(broker, vpnName);
402 InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
403 Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
404 if (dpnInVpn.isPresent()) {
405 List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
406 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
407 org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
408 currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
410 if (vpnInterfaces.remove(currVpnInterface)) {
411 if (vpnInterfaces.isEmpty()) {
412 VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id, VpnUtil.DEFAULT_CALLBACK);
413 fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
415 VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
416 org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
417 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
418 new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
424 private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label) {
426 bgpManager.addPrefix(rd, prefix, nextHopIp, (int)label);
427 } catch(Exception e) {
428 LOG.error("Add prefix failed", e);
433 private InstanceIdentifier<VpnInterface> getWildCardPath() {
434 return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
438 protected void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
439 LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
440 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
441 String interfaceName = key.getName();
443 InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
444 Optional<VpnInterface> existingVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, interfaceId);
445 org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
446 InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
448 if (existingVpnInterface.isPresent() && interfaceState != null) {
449 processVpnInterfaceDown(interfaceName, interfaceState.getIfIndex(), false);
451 LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
455 protected void processVpnInterfaceDown(String interfaceName, int lPortTag, boolean isInterfaceStateDown) {
456 VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
457 if(vpnInterface == null) {
458 LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
461 String vpnName = vpnInterface.getVpnInstanceName();
462 InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
464 synchronized (interfaceName.intern()) {
465 removeAdjacenciesFromVpn(identifier, vpnInterface);
466 LOG.info("Unbinding vpn service from interface {} ", interfaceName);
467 unbindService(vpnName, interfaceName, lPortTag, isInterfaceStateDown);
469 //wait till DCN for removal of vpn interface in operational DS arrives
470 Runnable notifyTask = new VpnNotifyTask();
471 synchronized (interfaceName.intern()) {
472 vpnIntfMap.put(interfaceName, notifyTask);
473 synchronized (notifyTask) {
475 notifyTask.wait(VpnConstants.WAIT_TIME_IN_MILLISECONDS);
476 } catch (InterruptedException e) {
484 private void removeAdjacenciesFromVpn(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
486 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
487 Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
489 String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
490 if (adjacencies.isPresent()) {
491 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
493 if (!nextHops.isEmpty()) {
494 LOG.trace("NextHops are " + nextHops);
495 for (Adjacency nextHop : nextHops) {
496 VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
497 VpnUtil.getNextHopLabelKey(rd, nextHop.getIpAddress()));
498 /*VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
499 VpnUtil.getPrefixToInterfaceIdentifier(
500 VpnUtil.getVpnId(broker, intf.getVpnInstanceName()),
501 nextHop.getIpAddress()),
502 VpnUtil.DEFAULT_CALLBACK);*/
503 if (rd.equals(intf.getVpnInstanceName())) {
504 //this is an internal vpn - the rd is assigned to the vpn instance name;
505 //remove from FIB directly
506 removeFibEntryFromDS(intf.getVpnInstanceName(), nextHop.getIpAddress());
508 removePrefixFromBGP(rd, nextHop.getIpAddress());
516 private void unbindService(String vpnInstanceName, String vpnInterfaceName, int lPortTag, boolean isInterfaceStateDown) {
517 if (!isInterfaceStateDown) {
518 VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION,
519 InterfaceUtils.buildServiceId(vpnInterfaceName,
520 VpnConstants.L3VPN_SERVICE_IDENTIFIER),
521 VpnUtil.DEFAULT_CALLBACK);
523 long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
524 makeArpFlow(VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
525 vpnId, ArpReplyOrRequest.REQUEST, NwConstants.DEL_FLOW);
529 private void removePrefixFromBGP(String rd, String prefix) {
531 bgpManager.deletePrefix(rd, prefix);
532 } catch(Exception e) {
533 LOG.error("Delete prefix failed", e);
538 protected void update(InstanceIdentifier<VpnInterface> identifier,
539 VpnInterface original, VpnInterface update) {
540 LOG.trace("Update VPN Interface {} , original {}, update {}",
541 identifier, original, update);
542 String vpnName = original.getVpnInstanceName();
544 boolean vpnNameChanged = false;
545 String rd = getRouteDistinguisher(vpnName);
547 String newVpnName = update.getVpnInstanceName();
548 if(!vpnName.equals(newVpnName)) {
549 //VPN for this interface got changed.
550 //Remove the interface from old VPN and add it to new VPN
551 newRd = getRouteDistinguisher(newVpnName);
552 if(newRd.equals("")) {
553 LOG.warn("VPN Instance {} not found. Update operation aborted", newVpnName);
556 vpnNameChanged = true;
557 LOG.debug("New VPN Name for the interface {} is {}", newVpnName, original.getName());
560 List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
561 List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
562 if(vpnNameChanged && newAdjs != null && !newAdjs.isEmpty()) {
563 long label = VpnConstants.INVALID_ID;
564 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
565 Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
566 if (adjacencies.isPresent()) {
567 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
568 for(Adjacency nextHop : nextHops) {
569 label = nextHop.getLabel();
570 if(label == VpnConstants.INVALID_ID) {
571 //Generate label using ID Manager
572 label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
573 VpnUtil.getNextHopLabelKey(newRd, nextHop.getIpAddress()));
576 removePrefixFromBGP(rd, nextHop.getIpAddress());
578 removeFibEntryFromDS(vpnName, nextHop.getIpAddress());
580 //updatePrefixToBGP(newRd, nextHop, nextHopIp, label);
582 processVpnInterfaceAdjacencies(identifier, update);
583 VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, update);
585 } else if (oldAdjs != newAdjs) {
586 //handle both addition and removal of adjacencies
587 //currently, new adjacency may be an extra route
588 for (Adjacency adj : newAdjs) {
589 if (oldAdjs.contains(adj)) {
592 // add new adjacency - right now only extra route will hit this path
593 addNewAdjToVpnInterface(identifier, adj);
597 for (Adjacency adj : oldAdjs) {
598 delAdjFromVpnInterface(identifier, adj);
602 LOG.debug("No Update information is available for VPN Interface to proceed");
607 public void processArpRequest(IpAddress srcIP, PhysAddress srcMac, IpAddress targetIP, String srcInterface){
608 SendArpResponseInput input = new SendArpResponseInputBuilder().setInterface(srcInterface)
609 .setIpaddress(srcIP).setSrcIpAddress(targetIP).setMacaddress(srcMac).build();
610 final String msgFormat = String.format("Send ARP Response on interface %s to destination %s", srcInterface, srcIP);
611 Future<RpcResult<Void>> future = arpManager.sendArpResponse(input);
612 Futures.addCallback(JdkFutureAdapters.listenInPoolThread(future), new FutureCallback<RpcResult<Void>>() {
614 public void onFailure(Throwable error) {
615 LOG.error("Error - {}", msgFormat, error);
619 public void onSuccess(RpcResult<Void> result) {
620 if(!result.isSuccessful()) {
621 LOG.warn("Rpc call to {} failed", msgFormat, getErrorText(result.getErrors()));
623 LOG.debug("Successful RPC Result - {}", msgFormat);
629 private String getErrorText(Collection<RpcError> errors) {
630 StringBuilder errorText = new StringBuilder();
631 for(RpcError error : errors) {
632 errorText.append(",").append(error.getErrorType()).append("-")
633 .append(error.getMessage());
635 return errorText.toString();
638 private String getTunnelInterfaceFlowRef(BigInteger dpnId, short tableId, String ifName) {
639 return new StringBuilder().append(dpnId).append(tableId).append(ifName).toString();
644 public synchronized void addFibEntryToDS(String rd, String prefix,
645 String nexthop, int label) {
647 VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).
648 setNextHopAddress(nexthop).setLabel((long)label).build();
649 LOG.debug("Created vrfEntry for {} nexthop {} label {}", prefix, nexthop, label);
651 List<VrfEntry> vrfEntryList = new ArrayList<VrfEntry>();
652 vrfEntryList.add(vrfEntry);
654 InstanceIdentifierBuilder<VrfTables> idBuilder =
655 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
656 InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
658 VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
659 setVrfEntry(vrfEntryList).build();
661 VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
664 public synchronized void removeFibEntryFromDS(String rd, String prefix) {
666 LOG.debug("Removing fib entry with destination prefix {} from vrf table for rd {}", prefix, rd);
668 InstanceIdentifierBuilder<VrfEntry> idBuilder =
669 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).child(VrfEntry.class, new VrfEntryKey(prefix));
670 InstanceIdentifier<VrfEntry> vrfEntryId = idBuilder.build();
671 VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId, VpnUtil.DEFAULT_CALLBACK);
675 public synchronized void removeVrfFromDS(String rd) {
676 LOG.debug("Removing vrf table for rd {}", rd);
678 InstanceIdentifierBuilder<VrfTables> idBuilder =
679 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
680 InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
682 VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, VpnUtil.DEFAULT_CALLBACK);
686 protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
688 Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
690 if (optVpnInterface.isPresent()) {
691 VpnInterface currVpnIntf = optVpnInterface.get();
692 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
693 String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
694 InstanceIdentifier<Adjacencies> adjPath = identifier.augmentation(Adjacencies.class);
695 Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, adjPath);
697 VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
698 VpnUtil.getNextHopLabelKey((rd != null) ? rd : currVpnIntf.getVpnInstanceName(), prefix));
700 List<Adjacency> adjacencies;
701 if (optAdjacencies.isPresent()) {
702 adjacencies = optAdjacencies.get().getAdjacency();
704 //This code will not be hit since VM adjacency will always be there
705 adjacencies = new ArrayList<>();
708 adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIp(adj.getNextHopIp())
709 .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
711 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
712 VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug);
714 VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
715 addExtraRoute(adj.getIpAddress(), adj.getNextHopIp(), rd, currVpnIntf.getVpnInstanceName(), (int) label, currVpnIntf.getName());
721 protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
722 Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
724 if (optVpnInterface.isPresent()) {
725 VpnInterface currVpnIntf = optVpnInterface.get();
727 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
728 Optional<Adjacencies> optAdjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
729 if (optAdjacencies.isPresent()) {
730 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
732 if (!adjacencies.isEmpty()) {
733 String rd = getRouteDistinguisher(currVpnIntf.getVpnInstanceName());
734 LOG.trace("Adjacencies are " + adjacencies);
735 Iterator<Adjacency> adjIt = adjacencies.iterator();
736 while (adjIt.hasNext()) {
737 Adjacency adjElem = adjIt.next();
738 if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
739 VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
740 VpnUtil.getNextHopLabelKey(rd, adj.getIpAddress()));
743 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
744 VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug);
746 VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
748 delExtraRoute(adj.getIpAddress(), rd, currVpnIntf.getVpnInstanceName());
759 protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label, String intfName) {
761 //add extra route to vpn mapping; advertise with nexthop as tunnel ip
764 LogicalDatastoreType.OPERATIONAL,
765 VpnUtil.getVpnToExtrarouteIdentifier(
766 (rd != null) ? rd : routerID, destination),
767 VpnUtil.getVpnToExtraroute(destination, nextHop));
769 if(intfName != null && !intfName.isEmpty()) {
770 BigInteger dpnId = InterfaceUtils.getDpnForInterface(interfaceManager, intfName);
771 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
772 if (nextHopIp == null && !nextHopIp.isEmpty()) {
773 LOG.error("NextHop for interface {} is null. Failed adding extra route for prefix {}", intfName, destination);
779 addPrefixToBGP(rd, destination, nextHop, label);
781 // ### add FIB route directly
782 addFibEntryToDS(routerID, destination, nextHop, label);
786 protected void delExtraRoute(String destination, String rd, String routerID) {
788 removePrefixFromBGP(rd, destination);
790 // ### add FIB route directly
791 removeFibEntryFromDS(routerID, destination);
795 class VpnInterfaceOpListener extends org.opendaylight.vpnservice.mdsalutil.AbstractDataChangeListener<VpnInterface> {
797 public VpnInterfaceOpListener() {
798 super(VpnInterface.class);
802 protected void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface del) {
803 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
804 String interfaceName = key.getName();
806 //increment the vpn interface count in Vpn Instance Op Data
808 String rd = getRouteDistinguisher(del.getVpnInstanceName());
809 if(rd.isEmpty()) rd = del.getVpnInstanceName();
810 VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
811 if(vpnInstOp != null && vpnInstOp.getVpnInterfaceCount() != null) {
812 ifCnt = vpnInstOp.getVpnInterfaceCount();
815 LOG.trace("VpnInterfaceOpListener remove: interface name {} rd {} interface count in Vpn Op Instance {}", interfaceName, rd, ifCnt);
817 VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
818 VpnUtil.getVpnInstanceOpDataIdentifier(rd),
819 VpnUtil.updateIntfCntInVpnInstOpData(ifCnt - 1, rd), VpnUtil.DEFAULT_CALLBACK);
821 //TODO: Clean up the DPN List in Vpn Instance Op if ifCnt is zero
823 notifyTaskIfRequired(interfaceName);
826 private void notifyTaskIfRequired(String intfName) {
827 Runnable notifyTask = vpnIntfMap.remove(intfName);
828 if (notifyTask == null) {
831 executorService.execute(notifyTask);
835 protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
839 protected void add(InstanceIdentifier<VpnInterface> identifier, VpnInterface add) {
840 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
841 String interfaceName = key.getName();
843 //increment the vpn interface count in Vpn Instance Op Data
845 String rd = getRouteDistinguisher(add.getVpnInstanceName());
846 if(rd.isEmpty()) rd = add.getVpnInstanceName();
847 VpnInstanceOpDataEntry vpnInstOp = VpnUtil.getVpnInstanceOpData(broker, rd);
848 if(vpnInstOp != null && vpnInstOp.getVpnInterfaceCount() != null) {
849 ifCnt = vpnInstOp.getVpnInterfaceCount();
852 LOG.trace("VpnInterfaceOpListener add: interface name {} rd {} interface count in Vpn Op Instance {}", interfaceName, rd, ifCnt);
854 VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
855 VpnUtil.getVpnInstanceOpDataIdentifier(rd),
856 VpnUtil.updateIntfCntInVpnInstOpData(ifCnt + 1, rd), VpnUtil.DEFAULT_CALLBACK);
862 protected void updatePrefixesForDPN(BigInteger dpnId, UpdateRouteAction action) {
864 InstanceIdentifierBuilder<VpnInstances> idBuilder = InstanceIdentifier.builder(VpnInstances.class);
865 InstanceIdentifier<VpnInstances> vpnInstancesId = idBuilder.build();
866 Optional<VpnInstances> vpnInstances = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnInstancesId);
868 if (vpnInstances.isPresent()) {
869 List<VpnInstance> vpnInstanceList = vpnInstances.get().getVpnInstance();
870 Iterator<VpnInstance> vpnInstIter = vpnInstanceList.iterator();
871 while (vpnInstIter.hasNext()) {
872 VpnInstance vpnInstance = vpnInstIter.next();
873 VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
874 String rd = vpnConfig.getRouteDistinguisher();
876 InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
877 Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
878 if (dpnInVpn.isPresent()) {
879 if (action == UpdateRouteAction.ADVERTISE_ROUTE) {
880 fibManager.populateFibOnNewDpn(dpnId, VpnUtil
881 .getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
883 List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
884 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
885 for(org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
886 .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces vpnInterface : vpnInterfaces) {
887 InstanceIdentifier<VpnInterface> vpnIntfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getInterfaceName());
888 InstanceIdentifier<Adjacencies> path = vpnIntfId.augmentation(Adjacencies.class);
889 Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
891 if (adjacencies.isPresent()) {
892 List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
893 Iterator<Adjacency> adjacencyIterator = adjacencyList.iterator();
895 while (adjacencyIterator.hasNext()) {
896 Adjacency adjacency = adjacencyIterator.next();
898 if(action == UpdateRouteAction.ADVERTISE_ROUTE)
899 bgpManager.addPrefix(rd, adjacency.getIpAddress(), adjacency.getNextHopIp(), adjacency.getLabel().intValue());
900 else if(action == UpdateRouteAction.WITHDRAW_ROUTE)
901 bgpManager.deletePrefix(rd, adjacency.getIpAddress());
902 } catch (Exception e) {
903 LOG.error("Exception when updating prefix {} in vrf {} to BGP", adjacency.getIpAddress(), rd);
909 if (action == UpdateRouteAction.WITHDRAW_ROUTE) {
910 fibManager.cleanUpDpnForVpn(dpnId, VpnUtil
911 .getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);