2 * Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.groupbasedpolicy.renderer.vpp.policy.acl;
11 import java.util.ArrayList;
12 import java.util.HashMap;
13 import java.util.List;
16 import java.util.concurrent.Callable;
17 import java.util.concurrent.ExecutionException;
18 import java.util.concurrent.Executors;
19 import java.util.function.Predicate;
20 import java.util.stream.Collectors;
21 import java.util.stream.Stream;
23 import javax.annotation.Nonnull;
24 import javax.annotation.Nullable;
26 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
27 import org.opendaylight.groupbasedpolicy.renderer.util.AddressEndpointUtils;
28 import org.opendaylight.groupbasedpolicy.renderer.vpp.iface.InterfaceManager;
29 import org.opendaylight.groupbasedpolicy.renderer.vpp.iface.VppPathMapper;
30 import org.opendaylight.groupbasedpolicy.renderer.vpp.policy.PolicyContext;
31 import org.opendaylight.groupbasedpolicy.renderer.vpp.policy.RendererResolvedPolicy;
32 import org.opendaylight.groupbasedpolicy.renderer.vpp.policy.ResolvedRuleGroup;
33 import org.opendaylight.groupbasedpolicy.renderer.vpp.policy.acl.AccessListUtil.ACE_DIRECTION;
34 import org.opendaylight.groupbasedpolicy.renderer.vpp.util.GbpNetconfTransaction;
35 import org.opendaylight.groupbasedpolicy.renderer.vpp.util.KeyFactory;
36 import org.opendaylight.groupbasedpolicy.renderer.vpp.util.MountedDataBrokerProvider;
37 import org.opendaylight.groupbasedpolicy.renderer.vpp.util.VppIidFactory;
38 import org.opendaylight.groupbasedpolicy.util.EndpointUtils;
39 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.access.control.list.rev160708.access.lists.AclKey;
40 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.access.control.list.rev160708.access.lists.acl.AccessListEntries;
41 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.access.control.list.rev160708.access.lists.acl.access.list.entries.Ace;
42 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
43 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.InterfaceKey;
44 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.base_endpoint.rev160427.endpoints.address.endpoints.AddressEndpointKey;
45 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.base_endpoint.rev160427.has.absolute.location.absolute.location.location.type.ExternalLocationCase;
46 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.forwarding.l2_l3.rev170511.L2BridgeDomain;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.HasDirection.Direction;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.endpoints.AddressEndpointWithLocation;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.renderer.endpoints.RendererEndpoint;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.renderer.endpoints.RendererEndpointKey;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.renderer.endpoints.renderer.endpoint.PeerEndpointKey;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.rule.groups.RuleGroupKey;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.vpp.acl.rev170615.VppAcl;
54 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
55 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
56 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
57 import org.slf4j.Logger;
58 import org.slf4j.LoggerFactory;
60 import com.google.common.base.Optional;
61 import com.google.common.base.Preconditions;
62 import com.google.common.collect.HashBasedTable;
63 import com.google.common.collect.HashMultimap;
64 import com.google.common.collect.ImmutableList;
65 import com.google.common.collect.ImmutableMap;
66 import com.google.common.collect.ImmutableSet;
67 import com.google.common.collect.ImmutableSetMultimap;
68 import com.google.common.collect.ImmutableSortedSet;
69 import com.google.common.collect.ImmutableTable;
70 import com.google.common.collect.ImmutableTable.Builder;
71 import com.google.common.collect.SetMultimap;
72 import com.google.common.collect.Sets;
73 import com.google.common.collect.Sets.SetView;
74 import com.google.common.collect.Table;
75 import com.google.common.util.concurrent.FutureCallback;
76 import com.google.common.util.concurrent.Futures;
77 import com.google.common.util.concurrent.ListenableFuture;
78 import com.google.common.util.concurrent.ListeningExecutorService;
79 import com.google.common.util.concurrent.MoreExecutors;
80 import com.google.common.util.concurrent.SettableFuture;
82 public class AclManager {
84 private static final Logger LOG = LoggerFactory.getLogger(AclManager.class);
85 private final MountedDataBrokerProvider mountDataProvider;
87 private static ImmutableTable<NodeId, InterfaceKey, ImmutableSet<AddressEndpointKey>> endpointsByInterface;
88 private final InterfaceManager interfaceManager;
89 ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(30));
91 public AclManager(@Nonnull MountedDataBrokerProvider mountDataProvider, InterfaceManager interfaceManager) {
92 this.mountDataProvider = Preconditions.checkNotNull(mountDataProvider);
93 this.interfaceManager = Preconditions.checkNotNull(interfaceManager);
96 public ListenableFuture<List<AccessListWrapper>> resolveAclsOnInterface(RendererEndpointKey rEpKey,
98 Callable<List<AccessListWrapper>> aclBuildExecutor = new Callable<List<AccessListWrapper>>() {
101 public List<AccessListWrapper> call() throws Exception {
102 LOG.info("Resolving ACL for renderer endpoint {}", rEpKey);
103 List<AccessListWrapper> aclWrappers = new ArrayList<>();
104 for (ACE_DIRECTION dir : new ACE_DIRECTION[] {ACE_DIRECTION.INGRESS, ACE_DIRECTION.EGRESS}) {
105 aclWrappers.add(buildAccessListWrappers(dir, ctx, rEpKey));
110 ListenableFuture<List<AccessListWrapper>> accessListFuture = executor.submit(aclBuildExecutor);
111 return accessListFuture;
115 * @param policyDirection direction for which policy should be resolved. EP -> VPP = OUTBOUND, EP <- VPP = INBOUND
116 * @param ctx with cached data
117 * @param rEpKey key of EP for which to create ACLs.
118 * @return synchronization futures, so that INGRESS and EGRESS ACLS can be resolved in parallel.
120 private static AccessListWrapper buildAccessListWrappers(ACE_DIRECTION policyDirection, PolicyContext ctx,
121 RendererEndpointKey rEpKey) {
122 LOG.trace("Resolving policy for VPP renderer endpoint {} in a separate thread in {} direction.", rEpKey,
124 AccessListWrapper aclWrapper = AccessListUtil.ACE_DIRECTION.INGRESS
125 .equals(policyDirection) ? new IngressAccessListWrapper() : new EgressAccessListWrapper();
126 AccessListUtil.configureLocalRules(ctx, rEpKey, policyDirection, aclWrapper);
127 // we support multiple IP end-points on a same interface
128 for (AddressEndpointKey aek : otherEndpointsOnTheSameInterface(ctx,
129 AddressEndpointUtils.fromRendererEpKey(rEpKey))) {
130 AccessListUtil.configureLocalRules(ctx, AddressEndpointUtils.toRendererEpKey(aek), policyDirection, aclWrapper);
132 // resolve peers with no location
133 aclWrapper.writeRules(AccessListUtil.denyDomainSubnets(ctx, policyDirection));
134 // TODO currently any traffic heading to/from outside of managed domain is
135 // permitted for demonstration purposes
136 if (rEpKey.getContextType().isAssignableFrom(L2BridgeDomain.class) && AccessListUtil.findAddrEp(ctx, rEpKey) != null) {
137 Optional<GbpAceBuilder> allowExtAccess =
138 AccessListUtil.allowExternalNetworksForEp(AccessListUtil.findAddrEp(ctx, rEpKey), policyDirection);
139 if (allowExtAccess.isPresent()) {
140 aclWrapper.writeRule(allowExtAccess.get());
146 public void resolveRulesToConfigure(@Nonnull PolicyContext policyCtx,
147 @Nonnull SetView<RendererEndpointKey> changedEndpoints, @Nonnull SetView<RuleGroupKey> changedRules, boolean write) {
148 Table<NodeId, AclKey, List<Ace>> aceTable = HashBasedTable.<NodeId, AclKey, List<Ace>>create();
149 Map<RendererEndpointKey, Set<RuleGroupKey>> endpointsToUpdate = new HashMap<>();
151 changedRules.forEach(changedGroupRuleKey -> {
152 policyCtx.getEndpointsByRuleGroups().get(changedGroupRuleKey).forEach(rendEp -> {
153 endpointsToUpdate.put(rendEp, changedRules);
156 // end-points changed
157 for (RendererEndpointKey rendEp : changedEndpoints) {
158 policyCtx.getPolicyTable().row(rendEp).keySet().forEach(peer -> {
159 ImmutableSortedSet<RendererResolvedPolicy> resolvedPolicy =
160 policyCtx.getPolicyTable().get(rendEp, peer);
161 if (resolvedPolicy == null) {
164 Set<RuleGroupKey> ruleGroupsToUpdate = resolvedPolicy.stream()
165 .map(policy -> policy.getRuleGroup().getRelatedRuleGroupKey())
166 .collect(Collectors.toSet());
167 if (endpointsToUpdate.get(rendEp) != null) {
168 endpointsToUpdate.put(rendEp,
169 Stream.concat(endpointsToUpdate.get(rendEp).stream(), ruleGroupsToUpdate.stream())
170 .collect(Collectors.toSet()));
172 endpointsToUpdate.put(rendEp, ruleGroupsToUpdate);
176 for (ACE_DIRECTION aceDirection : new ACE_DIRECTION[] {ACE_DIRECTION.INGRESS, ACE_DIRECTION.EGRESS}) {
177 endpointsToUpdate.forEach((endpointKey, rulesToUpdate) -> {
178 java.util.Optional<RendererEndpoint> endpointToUpdate = policyCtx.getPolicy()
180 .getRendererEndpoints()
181 .getRendererEndpoint()
183 .filter(rendEpInPolicy -> rendEpInPolicy.getKey().equals(endpointKey))
185 Builder<RendererEndpointKey, PeerEndpointKey, List<RendererResolvedPolicy>> updateTreeBuilder = new Builder<>();
186 endpointToUpdate.get()
189 .filter(peer -> policyCtx.getPolicyTable().get(endpointKey, peer.getKey()) != null)
190 .forEach(peer -> {updateTreeBuilder.put(endpointKey, peer.getKey(),
191 policyCtx.getPolicyTable()
192 .get(endpointKey, peer.getKey())
194 .filter(rrp -> rulesToUpdate.contains(rrp.getRuleGroup().getRelatedRuleGroupKey()))
195 .collect(Collectors.toList()));
197 ImmutableTable<RendererEndpointKey, PeerEndpointKey, List<RendererResolvedPolicy>> updateTree = updateTreeBuilder.build();
198 List<GbpAceBuilder> aceBuilders = new ArrayList<>();
199 updateTree.columnKeySet().stream().filter(p -> updateTree.get(endpointKey, p) != null).forEach(peer -> {
200 updateTree.get(endpointKey, peer).stream().forEach(rendererResolvedPolicy -> {
202 aceBuilders.addAll(generateRulesForEndpointPair(policyCtx, endpointKey, peer,
203 rendererResolvedPolicy, aceDirection));
205 // we only need to resolve rule names when removing ACE
206 aceBuilders.addAll(rendererResolvedPolicy.getRuleGroup()
209 .map(rule -> new GbpAceBuilder(
210 AccessListUtil.resolveAceName(rule.getName(), endpointKey, peer)))
211 .collect(Collectors.toList()));
215 ImmutableSetMultimap<NodeId, InterfaceKey> interfacesForEndpoint =
216 getInterfacesForEndpoint(policyCtx, KeyFactory.addressEndpointKey(endpointKey));
217 interfacesForEndpoint.keySet().forEach(nodeId -> {
218 ImmutableSet<InterfaceKey> intfcsOnNode = interfacesForEndpoint.get(nodeId);
219 intfcsOnNode.forEach(intfKey -> {
220 Optional<String> intfName =
221 VppPathMapper.interfacePathToInterfaceName(intfKey.getName());
222 Preconditions.checkArgument(intfName.isPresent(), "Failed to resolve interface name from " + intfKey);
223 if (aceTable.get(nodeId, intfKey) != null) {
224 List<Ace> aces = Stream
225 .concat(aceTable.get(nodeId, intfKey).stream(),
226 aceBuilders.stream().map(aceBuilder -> aceBuilder.build()))
227 .collect(Collectors.toList());
228 aceTable.put(nodeId, new AclKey(intfName.get() + aceDirection, VppAcl.class), aces);
230 aceTable.put(nodeId, new AclKey(intfName.get() + aceDirection, VppAcl.class),
232 .map(aceBuilder -> aceBuilder.build())
233 .collect(Collectors.toList()));
239 updateRules(ImmutableTable.copyOf(aceTable), write);
242 private void updateRules(ImmutableTable<NodeId, AclKey, List<Ace>> rulesToUpdate, boolean write) {
243 List<ListenableFuture<Void>> sync = new ArrayList<>();
244 rulesToUpdate.rowKeySet().forEach(nodeId -> {
245 Callable<Void> syncExecutor = new Callable<Void>() {
248 public Void call() throws Exception {
249 InstanceIdentifier<Node> vppIid = VppIidFactory.getNetconfNodeIid(nodeId);
250 Optional<DataBroker> dataBroker =
251 mountDataProvider.resolveDataBrokerForMountPoint(vppIid);
252 if (dataBroker.isPresent()) {
253 LOG.error("Failed to update ACLs for endpoints on node {}. Mount point does not exist.",
256 ImmutableMap<AclKey, List<Ace>> row = rulesToUpdate.row(nodeId);
257 row.keySet().forEach(aclKey -> {
258 Map<InstanceIdentifier<Ace>, Ace> entries = new HashMap<>();
259 row.get(aclKey).forEach(ace -> {
260 entries.put(VppIidFactory.getVppAcl(aclKey)
262 .child(AccessListEntries.class)
263 .child(Ace.class, ace.getKey())
266 if (entries.isEmpty()) {
269 LOG.debug("Updating ACL: Action={}, Node={}, ACL={}", write, nodeId.getValue(),
270 aclKey.getAclName());
271 boolean result = (write) ? GbpNetconfTransaction.netconfSyncedWrite(vppIid, entries,
272 GbpNetconfTransaction.RETRY_COUNT) : GbpNetconfTransaction.netconfSyncedDelete(
273 vppIid, entries.keySet(), GbpNetconfTransaction.RETRY_COUNT);
275 LOG.error("Failed to remove rules from ACL {} on mount point {}", aclKey,
282 sync.add(executor.submit(syncExecutor));
285 Futures.allAsList(sync).get();
286 } catch (InterruptedException | ExecutionException e) {
287 LOG.error("Failed to sync ACLs on VPP nodes. {}", e);
291 private List<GbpAceBuilder> generateRulesForEndpointPair(PolicyContext ctx, RendererEndpointKey r,
292 PeerEndpointKey p, RendererResolvedPolicy rrp, ACE_DIRECTION aceDirection) {
293 List<GbpAceBuilder> rules = new ArrayList<>();
294 Direction direction =
295 AccessListUtil.calculateClassifDirection(rrp.getRendererEndpointParticipation(), aceDirection);
296 ResolvedRuleGroup resolvedRuleGroup = ctx.getRuleGroupByKey().get(rrp.getRuleGroup().getRelatedRuleGroupKey());
297 resolvedRuleGroup.getRules().forEach(rule -> {
298 Optional<GbpAceBuilder> ace = AccessListUtil.resolveAceClassifersAndAction(rule, direction,
299 AccessListUtil.resolveAceName(rule.getName(), r, p));
300 if (ace.isPresent()) {
301 rules.add(ace.get());
304 AccessListUtil.updateAddressesInRules(rules, r, p, ctx, aceDirection, true);
309 public void updateAclsForPeers(PolicyContext policyCtx, RendererEndpointKey rEpKey) {
310 ImmutableSet<PeerEndpointKey> peers = policyCtx.getPolicyTable().row(rEpKey).keySet();
311 List<ListenableFuture<Void>> sync = new ArrayList<>();
312 for (RendererEndpointKey peerRendEp : peers.stream()
313 .map(AddressEndpointUtils::fromPeerEpKey)
314 .collect(Collectors.toList())
316 .map(AddressEndpointUtils::toRendererEpKey)
317 .collect(Collectors.toList())) {
318 sync.add(updateAclsForRendEp(peerRendEp, policyCtx));
321 Futures.allAsList(sync).get();
322 } catch (InterruptedException | ExecutionException e) {
323 LOG.error("Failed to update ACLs for peers of {}. {}", rEpKey, e);
327 public ListenableFuture<Void> updateAclsForRendEp(RendererEndpointKey rEpKey, PolicyContext policyCtx) {
328 SettableFuture<Void> sf = SettableFuture.create();
329 AddressEndpointWithLocation peerAddrEp = policyCtx.getAddrEpByKey().get(KeyFactory.addressEndpointKey(rEpKey));
330 ExternalLocationCase epLoc;
332 epLoc = InterfaceManager.resolveAndValidateLocation(peerAddrEp);
333 } catch (NullPointerException | IllegalArgumentException e) {
334 //TODO investigate, don't just move on.
335 LOG.warn("Peer {} has no location. Moving on...", peerAddrEp, e.getMessage());
336 return Futures.immediateFuture(null);
338 InstanceIdentifier<Node> vppNodeIid = (InstanceIdentifier<Node>) epLoc.getExternalNodeMountPoint();
339 Optional<InstanceIdentifier<Interface>> optInterfaceIid =
340 VppPathMapper.interfaceToInstanceIdentifier(epLoc.getExternalNodeConnector());
341 if (!optInterfaceIid.isPresent()) {
342 LOG.warn("Cannot find interface for endpoint {}. ACLs for endpoint not updated {}. ", rEpKey);
343 return Futures.immediateFuture(null);
345 if (interfaceManager.isExcludedFromPolicy(vppNodeIid.firstKeyOf(Node.class).getNodeId(),
346 optInterfaceIid.get().firstKeyOf(Interface.class).getName())) {
347 return Futures.immediateFuture(null);
349 LOG.info("Updating policy for endpoint {}", rEpKey);
350 ListenableFuture<List<AccessListWrapper>> future = resolveAclsOnInterface(rEpKey, policyCtx);
351 Futures.addCallback(future, new FutureCallback<List<AccessListWrapper>>() {
354 public void onSuccess(List<AccessListWrapper> result) {
356 acl -> acl.writeAcl(vppNodeIid, optInterfaceIid.get().firstKeyOf(Interface.class)));
361 public void onFailure(Throwable t) {
362 LOG.error("Failed to update ACL for interface {} on node {}",
363 optInterfaceIid.get().firstKeyOf(Interface.class), vppNodeIid.firstKeyOf(Node.class));
366 }, MoreExecutors.directExecutor());
371 * Cache end-points accessible via a single interface for further processing.
373 * @param ctx policy context
375 public void cacheMultiInterfaces(@Nonnull PolicyContext ctx) {
376 Builder<NodeId, InterfaceKey, ImmutableSet<AddressEndpointKey>> resultBuilder = new Builder<>();
377 resolveEndpointsOnMultipleInterface(ImmutableList.copyOf(ctx.getAddrEpByKey().values()), resultBuilder);
378 endpointsByInterface = resultBuilder.build();
382 * Recursively grouping interfaces behind the same port
384 private void resolveEndpointsOnMultipleInterface(@Nullable List<AddressEndpointWithLocation> eps,
385 @Nonnull Builder<NodeId, InterfaceKey, ImmutableSet<AddressEndpointKey>> builder) {
386 if (eps == null || eps.isEmpty()) {
389 // look for any end-point with absolute location as reference end-point in this cycle;
390 java.util.Optional<AddressEndpointWithLocation> refEndpoint =
391 eps.stream().filter(ep -> EndpointUtils.getExternalLocationFrom(ep).isPresent()).findAny();
392 if (!refEndpoint.isPresent()) {
395 Predicate<AddressEndpointWithLocation> sameLocation = new Predicate<AddressEndpointWithLocation>() {
397 public boolean test(AddressEndpointWithLocation addrEp) {
398 return AddressEndpointUtils.sameExternalLocationCase(refEndpoint.get(), addrEp);
401 Optional<ExternalLocationCase> extLoc = EndpointUtils.getExternalLocationFrom(refEndpoint.get());
402 Set<AddressEndpointKey> sameLocations = eps.stream()
403 .filter(sameLocation)
404 .map(addrEp -> AddressEndpointUtils.fromAddressEndpointWithLocationKey(addrEp.getKey()))
405 .collect(Collectors.toSet());
406 builder.put(extLoc.get().getExternalNodeMountPoint().firstKeyOf(Node.class).getNodeId(),
407 new InterfaceKey(extLoc.get().getExternalNodeConnector()),
408 ImmutableSet.<AddressEndpointKey>copyOf(sameLocations));
409 List<AddressEndpointWithLocation> differentLocations = eps.stream()
410 // keep end-points with different location and end-points with relative location in loop
411 .filter(sameLocation.negate().or(p -> !EndpointUtils.getExternalLocationFrom(p).isPresent()))
412 .collect(Collectors.toList());
413 if (!differentLocations.isEmpty()) {
414 resolveEndpointsOnMultipleInterface(differentLocations, builder);
418 public @Nonnull static ImmutableSet<AddressEndpointKey> otherEndpointsOnTheSameInterface(@Nonnull PolicyContext ctx,
419 @Nonnull AddressEndpointKey key) {
420 if (endpointsByInterface != null) {
421 for (InterfaceKey ifaceKey : endpointsByInterface.columnKeySet()) {
422 for (NodeId nodeId : endpointsByInterface.column(ifaceKey).keySet()) {
423 ImmutableSet<AddressEndpointKey> addrEps = endpointsByInterface.get(nodeId, ifaceKey);
424 if (addrEps != null && addrEps.contains(key) && addrEps.size() > 1) {
425 return endpointsByInterface.get(nodeId, ifaceKey);
430 return ImmutableSet.copyOf(Sets.newHashSet());
433 public @Nonnull static ImmutableSetMultimap<NodeId, InterfaceKey> getInterfacesForEndpoint(@Nonnull PolicyContext ctx,
434 @Nonnull AddressEndpointKey key) {
435 SetMultimap<NodeId, InterfaceKey> interfaces = HashMultimap.create();
436 if (endpointsByInterface != null) {
437 for (InterfaceKey ifaceKey : endpointsByInterface.columnKeySet()) {
438 for (NodeId nodeId : endpointsByInterface.column(ifaceKey).keySet()) {
439 ImmutableSet<AddressEndpointKey> addrEps = endpointsByInterface.get(nodeId, ifaceKey);
440 if (addrEps != null && addrEps.contains(key)) {
441 interfaces.put(nodeId, ifaceKey);
446 return ImmutableSetMultimap.copyOf(interfaces);