vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/vpnservice/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2015 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.vpnservice;
   9
  10 import java.math.BigInteger;
  11 import java.util.Collection;
  12 import java.util.Collections;
  13 import java.util.HashSet;
  14 import java.util.List;
  15 import java.util.ArrayList;
  16 import java.util.Map;
  17 import java.util.concurrent.ConcurrentHashMap;
  18 import java.util.concurrent.ExecutionException;
  19 import java.util.concurrent.Future;
  20
  21 import com.google.common.base.Optional;
  22 import com.google.common.collect.ImmutableList;
  23 import com.google.common.util.concurrent.Futures;
  24 import com.google.common.util.concurrent.FutureCallback;
  25
  26 import org.opendaylight.bgpmanager.api.IBgpManager;
  27 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  28 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  29 import org.opendaylight.fibmanager.api.IFibManager;
  30 import org.opendaylight.vpnservice.interfacemgr.interfaces.IInterfaceManager;
  31 import org.opendaylight.vpnservice.mdsalutil.FlowEntity;
  32 import org.opendaylight.vpnservice.mdsalutil.InstructionInfo;
  33 import org.opendaylight.vpnservice.mdsalutil.InstructionType;
  34 import org.opendaylight.vpnservice.mdsalutil.MDSALUtil;
  35 import org.opendaylight.vpnservice.mdsalutil.MatchFieldType;
  36 import org.opendaylight.vpnservice.mdsalutil.MatchInfo;
  37 import org.opendaylight.vpnservice.mdsalutil.MetaDataUtil;
  38 import org.opendaylight.vpnservice.mdsalutil.interfaces.IMdsalApiManager;
  39 import org.opendaylight.yangtools.concepts.ListenerRegistration;
  40 import org.opendaylight.yangtools.yang.binding.DataObject;
  41 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  42 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
  43 import org.opendaylight.yangtools.yang.common.RpcResult;
  44 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  45 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  46 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  47 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.Interfaces;
  49 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  50 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.InterfaceKey;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.AdjacencyList;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.Adjacency;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.VpnInstance1;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.GetUniqueIdInput;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.GetUniqueIdInputBuilder;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.GetUniqueIdOutput;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.IdManagerService;
  59 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.Adjacencies;
  61 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.AdjacenciesBuilder;
  69 import org.slf4j.Logger;
  70 import org.slf4j.LoggerFactory;
  71
  72 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
  73     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
  74     private ListenerRegistration<DataChangeListener> listenerRegistration;
  75     private final DataBroker broker;
  76     private final IBgpManager bgpManager;
  77     private IFibManager fibManager;
  78     private IMdsalApiManager mdsalManager;
  79     private IInterfaceManager interfaceManager;
  80     private IdManagerService idManager;
  81     private Map<Long, Collection<Long>> vpnToDpnsDb;
  82     private Map<Long, Collection<String>> dpnToInterfaceDb;
  83
  84     private static final FutureCallback<Void> DEFAULT_CALLBACK =
  85             new FutureCallback<Void>() {
  86                 public void onSuccess(Void result) {
  87                     LOG.debug("Success in Datastore operation");
  88                 }
  89
  90                 public void onFailure(Throwable error) {
  91                     LOG.error("Error in Datastore operation", error);
  92                 };
  93             };
  94
  95     /**
  96      * Responsible for listening to data change related to VPN Interface
  97      * Bind VPN Service on the interface and informs the BGP service
  98      *
  99      * @param db - dataBroker service reference
 100      */
 101     public VpnInterfaceManager(final DataBroker db, final IBgpManager bgpManager) {
 102         super(VpnInterface.class);
 103         broker = db;
 104         this.bgpManager = bgpManager;
 105         vpnToDpnsDb = new ConcurrentHashMap<>();
 106         dpnToInterfaceDb = new ConcurrentHashMap<>();
 107         registerListener(db);
 108     }
 109
 110     public void setMdsalManager(IMdsalApiManager mdsalManager) {
 111         this.mdsalManager = mdsalManager;
 112     }
 113
 114     public void setInterfaceManager(IInterfaceManager interfaceManager) {
 115         this.interfaceManager = interfaceManager;
 116     }
 117
 118     public void setFibManager(IFibManager fibManager) {
 119         this.fibManager = fibManager;
 120     }
 121
 122     public void setIdManager(IdManagerService idManager) {
 123         this.idManager = idManager;
 124     }
 125
 126     @Override
 127     public void close() throws Exception {
 128         if (listenerRegistration != null) {
 129             try {
 130                 listenerRegistration.close();
 131             } catch (final Exception e) {
 132                 LOG.error("Error when cleaning up DataChangeListener.", e);
 133             }
 134             listenerRegistration = null;
 135         }
 136         LOG.info("VPN Interface Manager Closed");
 137     }
 138
 139     private void registerListener(final DataBroker db) {
 140         try {
 141             listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 142                     getWildCardPath(), VpnInterfaceManager.this, DataChangeScope.SUBTREE);
 143         } catch (final Exception e) {
 144             LOG.error("VPN Service DataChange listener registration fail!", e);
 145             throw new IllegalStateException("VPN Service registration Listener failed.", e);
 146         }
 147     }
 148
 149     @Override
 150     protected void add(final InstanceIdentifier<VpnInterface> identifier,
 151             final VpnInterface vpnInterface) {
 152         LOG.trace("key: {} , value: {}", identifier, vpnInterface );
 153         addInterface(identifier, vpnInterface);
 154     }
 155
 156     private void addInterface(final InstanceIdentifier<VpnInterface> identifier,
 157                               final VpnInterface vpnInterface) {
 158         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 159         String interfaceName = key.getName();
 160         InstanceIdentifierBuilder<Interface> idBuilder =
 161                 InstanceIdentifier.builder(Interfaces.class).child(Interface.class, new InterfaceKey(interfaceName));
 162         InstanceIdentifier<Interface> id = idBuilder.build();
 163         Optional<Interface> port = read(LogicalDatastoreType.CONFIGURATION, id);
 164         if (port.isPresent()) {
 165             Interface interf = port.get();
 166             bindServiceOnInterface(interf, vpnInterface.getVpnInstanceName());
 167             updateNextHops(identifier, vpnInterface);
 168         }
 169     }
 170
 171     private void updateNextHops(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 172         //Read NextHops
 173         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 174         Optional<Adjacencies> adjacencies = read(LogicalDatastoreType.CONFIGURATION, path);
 175         String intfName = intf.getName();
 176
 177         if (adjacencies.isPresent()) {
 178             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 179             List<Adjacency> value = new ArrayList<>();
 180
 181             //Get the rd of the vpn instance
 182             String rd = getRouteDistinguisher(intf.getVpnInstanceName());
 183
 184             long dpnId = interfaceManager.getDpnForInterface(intfName);
 185             String nextHopIp = interfaceManager.getEndpointIpForDpn(dpnId);
 186
 187
 188             LOG.trace("NextHops are {}", nextHops);
 189             for (Adjacency nextHop : nextHops) {
 190                 String key = nextHop.getIpAddress();
 191                 long label = getUniqueId(key);
 192                 value.add(new AdjacencyBuilder(nextHop).setLabel(label).build());
 193             }
 194
 195             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
 196             VpnInterface opInterface = VpnUtil.getVpnInterface(intfName, intf.getVpnInstanceName(), aug);
 197             InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(intfName);
 198             asyncWrite(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, DEFAULT_CALLBACK);
 199             for (Adjacency nextHop : nextHops) {
 200                 String key = nextHop.getIpAddress();
 201                 long label = getUniqueId(key);
 202                 updatePrefixToBGP(rd, nextHop, nextHopIp, label);
 203             }
 204         }
 205     }
 206
 207     private Integer getUniqueId(String idKey) {
 208         GetUniqueIdInput getIdInput = new GetUniqueIdInputBuilder()
 209                                            .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
 210                                            .setIdKey(idKey).build();
 211
 212         try {
 213             Future<RpcResult<GetUniqueIdOutput>> result = idManager.getUniqueId(getIdInput);
 214             RpcResult<GetUniqueIdOutput> rpcResult = result.get();
 215             if(rpcResult.isSuccessful()) {
 216                 return rpcResult.getResult().getIdValue().intValue();
 217             } else {
 218                 LOG.warn("RPC Call to Get Unique Id returned with Errors {}", rpcResult.getErrors());
 219             }
 220         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 221             LOG.warn("Exception when getting Unique Id",e);
 222         }
 223         return 0;
 224     }
 225
 226     private long getVpnId(String vpnName) {
 227         //TODO: This should be a Util function
 228         InstanceIdentifier<VpnInstance1> id = InstanceIdentifier.builder(VpnInstances.class)
 229                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).augmentation(VpnInstance1.class).build();
 230         Optional<VpnInstance1> vpnInstance = read(LogicalDatastoreType.OPERATIONAL, id);
 231
 232         long vpnId = VpnConstants.INVALID_ID;
 233         if(vpnInstance.isPresent()) {
 234             vpnId = vpnInstance.get().getVpnId();
 235         }
 236         return vpnId;
 237     }
 238
 239     private String getRouteDistinguisher(String vpnName) {
 240         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
 241                                       .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
 242         Optional<VpnInstance> vpnInstance = read(LogicalDatastoreType.CONFIGURATION, id);
 243         String rd = "";
 244         if(vpnInstance.isPresent()) {
 245             VpnInstance instance = vpnInstance.get();
 246             VpnAfConfig config = instance.getIpv4Family();
 247             rd = config.getRouteDistinguisher();
 248         }
 249         return rd;
 250     }
 251
 252     private synchronized void updateMappingDbs(long vpnId, long dpnId, String intfName, String rd) {
 253         Collection<Long> dpnIds = vpnToDpnsDb.get(vpnId);
 254         if(dpnIds == null) {
 255             dpnIds = new HashSet<>();
 256         }
 257         if(dpnIds.add(dpnId)) {
 258             vpnToDpnsDb.put(vpnId, dpnIds);
 259             fibManager.populateFibOnNewDpn(dpnId, vpnId, rd);
 260         }
 261
 262         Collection<String> intfNames = dpnToInterfaceDb.get(dpnId);
 263         if(intfNames == null) {
 264             intfNames = new ArrayList<>();
 265         }
 266         intfNames.add(intfName);
 267         dpnToInterfaceDb.put(dpnId, intfNames);
 268     }
 269
 270     private synchronized void remoteFromMappingDbs(long vpnId, long dpnId, String inftName, String rd) {
 271         Collection<String> intfNames = dpnToInterfaceDb.get(dpnId);
 272         if(intfNames == null) {
 273             return;
 274         }
 275         intfNames.remove(inftName);
 276         dpnToInterfaceDb.put(dpnId, intfNames);
 277         //TODO: Delay 'DPN' removal so that other services can cleanup the entries for this dpn
 278         if(intfNames.isEmpty()) {
 279             Collection<Long> dpnIds = vpnToDpnsDb.get(vpnId);
 280             if(dpnIds == null) {
 281                 return;
 282             }
 283             dpnIds.remove(dpnId);
 284             vpnToDpnsDb.put(vpnId, dpnIds);
 285             fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
 286         }
 287     }
 288
 289     private void bindServiceOnInterface(Interface intf, String vpnName) {
 290         LOG.trace("Bind service on interface {} for VPN: {}", intf, vpnName);
 291
 292         long vpnId = getVpnId(vpnName);
 293         long dpId = interfaceManager.getDpnForInterface(intf.getName());
 294         if(dpId == 0L) {
 295             LOG.warn("DPN for interface {} not found. Bind service on this interface aborted.", intf.getName());
 296             return;
 297         } else {
 298             String rd = getRouteDistinguisher(vpnName);
 299             updateMappingDbs(vpnId, dpId, intf.getName(), rd);
 300         }
 301
 302         long portNo = interfaceManager.getPortForInterface(intf.getName());
 303         String flowRef = getVpnInterfaceFlowRef(dpId, VpnConstants.LPORT_INGRESS_TABLE, vpnId, portNo);
 304
 305         String flowName = intf.getName();
 306         BigInteger COOKIE_VM_INGRESS_TABLE = new BigInteger("8000001", 16);
 307
 308         int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 309         short gotoTableId = VpnConstants.FIB_TABLE;
 310
 311         List<InstructionInfo> mkInstructions = new ArrayList<InstructionInfo>();
 312         mkInstructions.add(new InstructionInfo(InstructionType.write_metadata, new BigInteger[] {
 313                 BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
 314
 315         mkInstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { gotoTableId }));
 316
 317         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 318         matches.add(new MatchInfo(MatchFieldType.in_port, new long[] {
 319                 dpId, portNo }));
 320
 321         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, VpnConstants.LPORT_INGRESS_TABLE, flowRef,
 322                           priority, flowName, 0, 0, COOKIE_VM_INGRESS_TABLE, matches, mkInstructions);
 323
 324         mdsalManager.installFlow(flowEntity);
 325     }
 326
 327     private String getVpnInterfaceFlowRef(long dpId, short tableId,
 328             long vpnId, long portNo) {
 329         return new StringBuilder().append(dpId).append(tableId).append(vpnId).append(portNo).toString();
 330     }
 331
 332     private void updatePrefixToBGP(String rd, Adjacency nextHop, String nextHopIp, long label) {
 333         try {
 334             bgpManager.addPrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
 335         } catch(Exception e) {
 336             LOG.error("Add prefix failed", e);
 337         }
 338     }
 339
 340     private <T extends DataObject> Optional<T> read(LogicalDatastoreType datastoreType,
 341             InstanceIdentifier<T> path) {
 342
 343         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 344
 345         Optional<T> result = Optional.absent();
 346         try {
 347             result = tx.read(datastoreType, path).get();
 348         } catch (Exception e) {
 349             throw new RuntimeException(e);
 350         }
 351
 352         return result;
 353     }
 354
 355     private InstanceIdentifier<VpnInterface> getWildCardPath() {
 356         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 357     }
 358
 359     @Override
 360     protected void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
 361         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
 362         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 363         String interfaceName = key.getName();
 364         InstanceIdentifierBuilder<Interface> idBuilder =
 365                 InstanceIdentifier.builder(Interfaces.class).child(Interface.class, new InterfaceKey(interfaceName));
 366         InstanceIdentifier<Interface> id = idBuilder.build();
 367         Optional<Interface> port = read(LogicalDatastoreType.CONFIGURATION, id);
 368         if (port.isPresent()) {
 369             Interface interf = port.get();
 370             removeNextHops(identifier, vpnInterface);
 371             unbindServiceOnInterface(interf, vpnInterface.getVpnInstanceName());
 372             //InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 373             delete(LogicalDatastoreType.OPERATIONAL, identifier);
 374         } else {
 375             LOG.warn("No nexthops were available to handle remove event {}", interfaceName);
 376         }
 377     }
 378
 379     private void removeNextHops(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
 380         //Read NextHops
 381         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 382         Optional<Adjacencies> adjacencies = read(LogicalDatastoreType.OPERATIONAL, path);
 383         String intfName = intf.getName();
 384         String rd = getRouteDistinguisher(intf.getVpnInstanceName());
 385         if (adjacencies.isPresent()) {
 386             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 387
 388             if (!nextHops.isEmpty()) {
 389                 LOG.trace("NextHops are " + nextHops);
 390                 for (Adjacency nextHop : nextHops) {
 391                     removePrefixFromBGP(rd, nextHop);
 392                 }
 393             }
 394         }
 395 //        InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(intfName);
 396 //        delete(LogicalDatastoreType.OPERATIONAL, interfaceId);
 397     }
 398
 399     private <T extends DataObject> void delete(LogicalDatastoreType datastoreType, InstanceIdentifier<T> path) {
 400         WriteTransaction tx = broker.newWriteOnlyTransaction();
 401         tx.delete(datastoreType, path);
 402         Futures.addCallback(tx.submit(), DEFAULT_CALLBACK);
 403     }
 404
 405     private void unbindServiceOnInterface(Interface intf, String vpnName) {
 406         LOG.trace("Unbind service on interface {} for VPN: {}", intf, vpnName);
 407
 408         long vpnId = getVpnId(vpnName);
 409         long dpId = interfaceManager.getDpnForInterface(intf.getName());
 410         if(dpId == 0L) {
 411             LOG.warn("DPN for interface {} not found. Unbind service on this interface aborted.", intf.getName());
 412             return;
 413         } else {
 414             String rd = getRouteDistinguisher(vpnName);
 415             remoteFromMappingDbs(vpnId, dpId, intf.getName(), rd);
 416         }
 417
 418         long portNo = interfaceManager.getPortForInterface(intf.getName());
 419         String flowRef = getVpnInterfaceFlowRef(dpId, VpnConstants.LPORT_INGRESS_TABLE, vpnId, portNo);
 420
 421         String flowName = intf.getName();
 422
 423         int priority = VpnConstants.DEFAULT_FLOW_PRIORITY;
 424
 425         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 426         matches.add(new MatchInfo(MatchFieldType.in_port, new long[] {
 427                 dpId, portNo }));
 428
 429         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, VpnConstants.LPORT_INGRESS_TABLE, flowRef,
 430                           priority, flowName, 0, 0, null, matches, null);
 431
 432         mdsalManager.removeFlow(flowEntity);
 433     }
 434
 435     private void removePrefixFromBGP(String rd, Adjacency nextHop) {
 436         //public void deletePrefix(String rd, String prefix) throws Exception;
 437         try {
 438             bgpManager.deletePrefix(rd, nextHop.getIpAddress());
 439         } catch(Exception e) {
 440             LOG.error("Delete prefix failed", e);
 441         }
 442     }
 443
 444     @Override
 445     protected void update(InstanceIdentifier<VpnInterface> identifier,
 446                                    VpnInterface original, VpnInterface update) {
 447         // TODO Auto-generated method stub
 448
 449     }
 450
 451     private <T extends DataObject> void asyncWrite(LogicalDatastoreType datastoreType,
 452                         InstanceIdentifier<T> path, T data, FutureCallback<Void> callback) {
 453         WriteTransaction tx = broker.newWriteOnlyTransaction();
 454         tx.put(datastoreType, path, data, true);
 455         Futures.addCallback(tx.submit(), callback);
 456     }
 457
 458     synchronized Collection<Long> getDpnsForVpn(long vpnId) {
 459         Collection<Long> dpnIds = vpnToDpnsDb.get(vpnId);
 460         if(dpnIds != null) {
 461             return ImmutableList.copyOf(dpnIds);
 462         } else {
 463             return Collections.emptyList();
 464         }
 465     }
 466 }