renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/arp/ArpTasker.java

   1 /*
   2  * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.arp;
  10
  11 import static com.google.common.base.Preconditions.checkNotNull;
  12
  13 import java.util.ArrayList;
  14 import java.util.List;
  15 import java.util.concurrent.Future;
  16
  17 import javax.annotation.Nullable;
  18
  19 import org.apache.commons.lang3.tuple.ImmutablePair;
  20 import org.apache.commons.lang3.tuple.Pair;
  21 import org.apache.commons.net.util.SubnetUtils;
  22 import org.apache.commons.net.util.SubnetUtils.SubnetInfo;
  23 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  24 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  25 import org.opendaylight.controller.md.sal.binding.api.ReadTransaction;
  26 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  28 import org.opendaylight.controller.sal.binding.api.RpcProviderRegistry;
  29 import org.opendaylight.groupbasedpolicy.util.DataStoreHelper;
  30 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  31 import org.opendaylight.openflowplugin.api.OFConstants;
  32 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  33 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  34 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNodeConnector;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowInputBuilder;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowOutput;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.RemoveFlowInput;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.RemoveFlowInputBuilder;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.SalFlowService;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowModFlags;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowRef;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.ApplyActionsCaseBuilder;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActions;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActionsBuilder;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.ContextId;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.L2BridgeDomainId;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.L2ContextId;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.L2FloodDomainId;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.SubnetId;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointKey;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Builder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContextBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayL3Context;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayNodeConfig;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.nodes.node.ExternalInterfaces;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L2BridgeDomain;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L2FloodDomain;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.Subnet;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnector;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnectorKey;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatch;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatch;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketProcessingListener;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketProcessingService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketReceived;
  92 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  93 import org.opendaylight.yangtools.yang.common.RpcResult;
  94 import org.slf4j.Logger;
  95 import org.slf4j.LoggerFactory;
  96
  97 import com.google.common.base.Optional;
  98 import com.google.common.collect.ArrayListMultimap;
  99 import com.google.common.collect.HashMultimap;
 100 import com.google.common.collect.ImmutableList;
 101 import com.google.common.collect.ImmutableSetMultimap;
 102 import com.google.common.collect.ListMultimap;
 103 import com.google.common.collect.Multimaps;
 104 import com.google.common.collect.SetMultimap;
 105 import com.google.common.util.concurrent.FutureCallback;
 106 import com.google.common.util.concurrent.Futures;
 107 import com.google.common.util.concurrent.JdkFutureAdapters;
 108 import com.google.common.util.concurrent.ListenableFuture;
 109
 110 public class ArpTasker implements PacketProcessingListener {
 111
 112     private static final Logger LOG = LoggerFactory.getLogger(ArpTasker.class);
 113     private static final short TABEL_FOR_ARP_FLOW = 0;
 114     private static final String ARP_REPLY_TO_CONTROLLER_FLOW_NAME = "arpReplyToController";
 115     private static final int ARP_REPLY_TO_CONTROLLER_FLOW_PRIORITY = 10000;
 116     private static final Instruction SEND_TO_CONTROLLER_INSTRUCTION;
 117     private final ArpSender arpSender;
 118     private final SalFlowService flowService;
 119     private final DataBroker dataProvider;
 120     private final ListMultimap<String, Pair<RemoveFlowInput, EndpointL3Key>> requestInfoByKey = Multimaps.synchronizedListMultimap(ArrayListMultimap.<String, Pair<RemoveFlowInput, EndpointL3Key>>create());
 121
 122     static {
 123         ApplyActions applyActions = new ApplyActionsBuilder().setAction(
 124                 ImmutableList.of(ArpFlowFactory.createSendToControllerAction(0))).build();
 125         SEND_TO_CONTROLLER_INSTRUCTION = new InstructionBuilder().setOrder(0)
 126             .setInstruction(new ApplyActionsCaseBuilder().setApplyActions(applyActions).build())
 127             .build();
 128     }
 129
 130     public ArpTasker(RpcProviderRegistry rpcRegistry, DataBroker dataProvider) {
 131         this.dataProvider = checkNotNull(dataProvider);
 132         checkNotNull(rpcRegistry);
 133         PacketProcessingService packetProcessingService = rpcRegistry.getRpcService(PacketProcessingService.class);
 134         if (packetProcessingService != null) {
 135             LOG.info("{} was found.", PacketProcessingService.class.getSimpleName());
 136             this.arpSender = new ArpSender(packetProcessingService);
 137         } else {
 138             LOG.info("Missing service {}", PacketProcessingService.class.getSimpleName());
 139             this.arpSender = null;
 140         }
 141         flowService = rpcRegistry.getRpcService(SalFlowService.class);
 142     }
 143
 144     @Override
 145     public void onPacketReceived(PacketReceived potentialArp) {
 146         Arp arp = null;
 147         try {
 148             arp = ArpResolverUtils.getArpFrom(potentialArp);
 149         } catch (Exception e) {
 150             LOG.trace(
 151                     "Failed to decode potential ARP packet. This could occur when other than ARP packet was received.",
 152                     e);
 153             return;
 154         }
 155         if (arp.getOperation() != ArpOperation.REPLY.intValue()) {
 156             LOG.trace("ARP packet is not REPLY.");
 157             return;
 158         }
 159         if (LOG.isTraceEnabled()) {
 160             LOG.trace("ARP REPLY received - {}", ArpUtils.getArpToStringFormat(arp));
 161         }
 162         NodeKey nodeKey = potentialArp.getIngress().getValue().firstKeyOf(Node.class, NodeKey.class);
 163         if (nodeKey == null) {
 164             LOG.info("Unknown source node of ARP packet: {}", potentialArp);
 165             return;
 166         }
 167         Ipv4Address spa = ArpUtils.bytesToIp(arp.getSenderProtocolAddress());
 168         MacAddress sha = ArpUtils.bytesToMac(arp.getSenderHardwareAddress());
 169         List<Pair<RemoveFlowInput, EndpointL3Key>> removeFlowInputsAndL3EpKeys = requestInfoByKey.get(createKey(nodeKey.getId(), spa));
 170         for (Pair<RemoveFlowInput, EndpointL3Key> removeFlowInputAndL3EpKey : removeFlowInputsAndL3EpKeys) {
 171             flowService.removeFlow(removeFlowInputAndL3EpKey.getLeft());
 172         }
 173         // each L3EpKey on right-side part of Pair is same
 174         final EndpointL3Key l3EpKey = removeFlowInputsAndL3EpKeys.get(0).getRight();
 175         ReadWriteTransaction rwTx = dataProvider.newReadWriteTransaction();
 176         InstanceIdentifier<EndpointL3> l3EpIid = IidFactory.l3EndpointIid(l3EpKey.getL3Context(),
 177                 l3EpKey.getIpAddress());
 178         Optional<EndpointL3> potentialL3Ep = DataStoreHelper.readFromDs(LogicalDatastoreType.OPERATIONAL, l3EpIid, rwTx);
 179         if (!potentialL3Ep.isPresent()) {
 180             LOG.info("L3 endpoint {} where MAC should be added does not exist anymore.", l3EpKey);
 181             rwTx.cancel();
 182             return;
 183         }
 184         final EndpointL3Builder updatedL3EpBuilder = new EndpointL3Builder(potentialL3Ep.get()).setMacAddress(sha)
 185             .setTimestamp(System.currentTimeMillis());
 186         EndpointL3 updatedL3Ep = updatedL3EpBuilder.build();
 187         L2BridgeDomainId l2BdId = resolveL2BridgeDomainId(updatedL3Ep, rwTx);
 188         if (l2BdId != null) {
 189             updatedL3Ep = updatedL3EpBuilder.setL2Context(l2BdId).build();
 190             EndpointBuilder newEpBuilder = new EndpointBuilder(updatedL3Ep).setKey(new EndpointKey(l2BdId, sha));
 191             OfOverlayL3Context augmentation = updatedL3Ep.getAugmentation(OfOverlayL3Context.class);
 192             if (augmentation != null) {
 193                 newEpBuilder.addAugmentation(OfOverlayContext.class, new OfOverlayContextBuilder(augmentation).build());
 194             }
 195             Endpoint newEp = newEpBuilder.build();
 196             rwTx.put(LogicalDatastoreType.OPERATIONAL, IidFactory.endpointIid(l2BdId, sha), newEp);
 197             LOG.trace("Endpoint was created {}", newEp);
 198         }
 199         rwTx.put(LogicalDatastoreType.OPERATIONAL, l3EpIid, updatedL3Ep);
 200         LOG.trace("MAC was added to L3 endpoint {}", updatedL3Ep);
 201         rwTx.submit();
 202     }
 203
 204     private @Nullable L2BridgeDomainId resolveL2BridgeDomainId(EndpointL3 l3Ep, ReadTransaction rTx) {
 205         TenantId tenantId = l3Ep.getTenant();
 206         Subnet subnetOfL3Ep = readSubnet(l3Ep, rTx);
 207         if (subnetOfL3Ep == null) {
 208             return null;
 209         }
 210         ContextId parentOfSubnet = subnetOfL3Ep.getParent();
 211         if (parentOfSubnet == null) {
 212             return null;
 213         }
 214         L2ContextId l2ContextId = new L2ContextId(parentOfSubnet);
 215         Optional<L2BridgeDomain> potentialL2Bd = DataStoreHelper.readFromDs(LogicalDatastoreType.OPERATIONAL,
 216                 IidFactory.l2BridgeDomainIid(tenantId, new L2BridgeDomainId(l2ContextId)), rTx);
 217         if (potentialL2Bd.isPresent()) {
 218             return potentialL2Bd.get().getId();
 219         }
 220         Optional<L2FloodDomain> potentialL2Fd = DataStoreHelper.readFromDs(LogicalDatastoreType.OPERATIONAL,
 221                 IidFactory.l2FloodDomainIid(tenantId, new L2FloodDomainId(l2ContextId)), rTx);
 222         if (!potentialL2Fd.isPresent()) {
 223             return null;
 224         }
 225         return potentialL2Fd.get().getParent();
 226     }
 227
 228     /**
 229      * Uses ARP to get MAC for the given L3 endpoint. Tries to find MAC for IP from
 230      * {@link EndpointL3#getKey()}.<br>
 231      * {@link EndpointL3#getNetworkContainment()} has to point to a {@link Subnet}.<br>
 232      * ARP Request is sent from all node connectors obtaining from
 233      * {@link OfOverlayNodeConfig#getExternalInterfaces()}<br>
 234      * MAC address obtained from ARP reply is added to the given L3 endpoint (if still exits).<br>
 235      * Also an {@link Endpoint} is created based on MAC If the subnet from network containment point
 236      * to {@link L2BridgeDomain} directly or throught {@link L2FloodDomain}.
 237      *
 238      * @param l3Ep the L3 endpoint which needs to have an MAC address
 239      */
 240     public void addMacForL3EpAndCreateEp(final EndpointL3 l3Ep) {
 241         final Ipv4Address tpa = getIPv4Addresses(l3Ep);
 242         if (tpa == null) {
 243             LOG.debug("L3 endpoint {} does not contain IPv4 address.", l3Ep.getKey());
 244             return;
 245         }
 246         ReadOnlyTransaction rTx = dataProvider.newReadOnlyTransaction();
 247         final SetMultimap<Node, Pair<InstanceIdentifier<NodeConnector>, MacAddress>> extNcWithMacByNode = readNodesWithExternalIfaces(rTx);
 248         if (extNcWithMacByNode.isEmpty()) {
 249             LOG.debug("No node with external interface was found.");
 250             rTx.close();
 251             return;
 252         }
 253         final Ipv4Address senderIpAddress = createSenderIpAddress(l3Ep, rTx);
 254         if (senderIpAddress == null) {
 255             LOG.warn("Cannot create sender IPv4 address for L3 endpoint {}", l3Ep);
 256             rTx.close();
 257             return;
 258         }
 259         rTx.close();
 260
 261         for (final Node node : extNcWithMacByNode.keySet()) {
 262             final InstanceIdentifier<Node> nodeIid = InstanceIdentifier.builder(Nodes.class)
 263                 .child(Node.class, node.getKey())
 264                 .build();
 265             final NodeRef nodeRef = new NodeRef(nodeIid);
 266             List<ListenableFuture<RpcResult<AddFlowOutput>>> arpFlowResultFutures = new ArrayList<>();
 267             List<Pair<RemoveFlowInput, EndpointL3Key>> flowsForRemove = new ArrayList<>();
 268             for (final Pair<InstanceIdentifier<NodeConnector>, MacAddress> extNcIidAndMac : extNcWithMacByNode.get(node)) {
 269                 final ArpMessageAddress senderAddress = new ArpMessageAddress(extNcIidAndMac.getRight(),
 270                         senderIpAddress);
 271                 NodeConnectorId ncId = extNcIidAndMac.getLeft().firstKeyOf(NodeConnector.class, NodeConnectorKey.class).getId();
 272                 final Flow arpReplyToControllerFlow = createArpReplyToControllerFlow(senderAddress, tpa, ncId);
 273                 flowsForRemove.add(new ImmutablePair<>(new RemoveFlowInputBuilder(arpReplyToControllerFlow).setNode(
 274                         nodeRef).build(), l3Ep.getKey()));
 275                 final InstanceIdentifier<Flow> flowIid = createFlowIid(arpReplyToControllerFlow, nodeIid);
 276                 Future<RpcResult<AddFlowOutput>> futureAddFlowResult = flowService.addFlow(new AddFlowInputBuilder(
 277                         arpReplyToControllerFlow).setFlowRef(new FlowRef(flowIid)).setNode(nodeRef).build());
 278                 arpFlowResultFutures.add(JdkFutureAdapters.listenInPoolThread(futureAddFlowResult));
 279             }
 280             requestInfoByKey.putAll(createKey(node.getId(), tpa), flowsForRemove);
 281             ListenableFuture<List<RpcResult<AddFlowOutput>>> futureArpFlowResults = Futures.allAsList(arpFlowResultFutures);
 282             Futures.addCallback(futureArpFlowResults, new FutureCallback<List<RpcResult<AddFlowOutput>>>() {
 283
 284                 @Override
 285                 public void onSuccess(List<RpcResult<AddFlowOutput>> result) {
 286                     for (RpcResult<AddFlowOutput> addFlowResult : result) {
 287                         if (!addFlowResult.isSuccessful()) {
 288                             LOG.warn("An ARP Reply to Controller flow was not created on node {} \nErrors: {}",
 289                                     node.getId().getValue(), addFlowResult.getErrors());
 290                             continue;
 291                         }
 292                     }
 293                     LOG.debug("ARP Reply to Controller flows were created on node {}", node.getId().getValue());
 294                     for (final Pair<InstanceIdentifier<NodeConnector>, MacAddress> extNcIidAndMac : extNcWithMacByNode.get(node)) {
 295                         final ArpMessageAddress senderAddress = new ArpMessageAddress(extNcIidAndMac.getRight(),
 296                                 senderIpAddress);
 297                         ListenableFuture<RpcResult<Void>> futureSendArpResult = arpSender.sendArp(senderAddress, tpa,
 298                                 extNcIidAndMac.getLeft());
 299                         Futures.addCallback(futureSendArpResult, logResult(tpa, extNcIidAndMac.getLeft()));
 300                     }
 301                 }
 302
 303                 @Override
 304                 public void onFailure(Throwable t) {
 305                     LOG.error(
 306                             "Illegal state - Installation of ARP flows on node {} failed. Node can contain just some ARP flows.",
 307                             node.getId(), t);
 308                 }
 309             });
 310         }
 311     }
 312
 313     private static @Nullable Ipv4Address getIPv4Addresses(EndpointL3 l3ep) {
 314         IpAddress ipAddress = l3ep.getKey().getIpAddress();
 315         if (ipAddress.getIpv4Address() == null) {
 316             return null;
 317         }
 318         return ipAddress.getIpv4Address();
 319     }
 320
 321     private SetMultimap<Node, Pair<InstanceIdentifier<NodeConnector>, MacAddress>> readNodesWithExternalIfaces(
 322             ReadTransaction rTx) {
 323         Optional<Nodes> potentialNodes = DataStoreHelper.readFromDs(LogicalDatastoreType.CONFIGURATION,
 324                 InstanceIdentifier.builder(Nodes.class).build(), rTx);
 325         if (!potentialNodes.isPresent() || potentialNodes.get().getNode() == null) {
 326             return ImmutableSetMultimap.of();
 327         }
 328         List<Node> nodes = potentialNodes.get().getNode();
 329         SetMultimap<Node, Pair<InstanceIdentifier<NodeConnector>, MacAddress>> extIfacesByNode = HashMultimap.create();
 330         for (Node node : nodes) {
 331             OfOverlayNodeConfig ofOverlayNode = node.getAugmentation(OfOverlayNodeConfig.class);
 332             if (ofOverlayNode != null) {
 333                 List<ExternalInterfaces> externalIfaces = ofOverlayNode.getExternalInterfaces();
 334                 if (externalIfaces != null) {
 335                     for (ExternalInterfaces extIface : externalIfaces) {
 336                         NodeConnectorId externalNc = extIface.getNodeConnectorId();
 337                         InstanceIdentifier<NodeConnector> extNcIid = InstanceIdentifier.builder(Nodes.class)
 338                             .child(Node.class, node.getKey())
 339                             .child(NodeConnector.class, new NodeConnectorKey(externalNc))
 340                             .build();
 341                         Optional<NodeConnector> potentialExtNcFromOper = DataStoreHelper.readFromDs(
 342                                 LogicalDatastoreType.OPERATIONAL, extNcIid, rTx);
 343                         if (!potentialExtNcFromOper.isPresent()) {
 344                             LOG.debug("Node connector {} does not exit in OPER DS. Node from CONF: {}",
 345                                     externalNc.getValue(), node);
 346                             continue;
 347                         }
 348                         FlowCapableNodeConnector externalFcNc = potentialExtNcFromOper.get().getAugmentation(
 349                                 FlowCapableNodeConnector.class);
 350                         if (externalFcNc == null || externalFcNc.getHardwareAddress() == null) {
 351                             LOG.debug("Hardware address does not exist on node connector {}", externalNc.getValue());
 352                             LOG.trace("Node connector from OPER DS {}", potentialExtNcFromOper.get());
 353                             continue;
 354                         }
 355                         extIfacesByNode.put(node, new ImmutablePair<>(extNcIid, externalFcNc.getHardwareAddress()));
 356                     }
 357                 }
 358             }
 359         }
 360         return extIfacesByNode;
 361     }
 362
 363     private @Nullable Ipv4Address createSenderIpAddress(EndpointL3 l3Ep, ReadTransaction rTx) {
 364         Subnet subnetOfL3Ep = readSubnet(l3Ep, rTx);
 365         if (subnetOfL3Ep == null) {
 366             return null;
 367         }
 368         SubnetInfo subnetInfo = new SubnetUtils(subnetOfL3Ep.getIpPrefix().getIpv4Prefix().getValue()).getInfo();
 369         String senderIp = subnetInfo.getHighAddress();
 370         if (senderIp.equals(l3Ep.getKey().getIpAddress().getIpv4Address().getValue())) {
 371             senderIp = subnetInfo.getLowAddress();
 372         }
 373         return new Ipv4Address(senderIp);
 374     }
 375
 376     private @Nullable Subnet readSubnet(EndpointL3 l3Ep, ReadTransaction rTx) {
 377         NetworkDomainId l3EpNetworkContainment = l3Ep.getNetworkContainment();
 378         if (l3EpNetworkContainment == null) {
 379             LOG.debug("L3 endpoint {} does not contain network containment.", l3Ep.getKey());
 380             return null;
 381         }
 382         if (l3Ep.getTenant() == null) {
 383             LOG.debug("L3 endpoint {} does not contain tenat.", l3Ep.getKey());
 384             return null;
 385         }
 386         Optional<Subnet> potentialSubnet = DataStoreHelper.readFromDs(LogicalDatastoreType.CONFIGURATION,
 387                 IidFactory.subnetIid(l3Ep.getTenant(), new SubnetId(l3EpNetworkContainment)), rTx);
 388         if (!potentialSubnet.isPresent()) {
 389             LOG.debug(
 390                     "Network containment {} of L3 endpoint {} does not point to a subnet or the subnet does not exist.",
 391                     l3EpNetworkContainment.getValue(), l3Ep.getKey());
 392             return null;
 393         }
 394         return potentialSubnet.get();
 395     }
 396
 397     private Flow createArpReplyToControllerFlow(ArpMessageAddress senderAddress, Ipv4Address ipForRequestedMac, NodeConnectorId inPort) {
 398         checkNotNull(senderAddress);
 399         checkNotNull(ipForRequestedMac);
 400         FlowBuilder arpFlow = new FlowBuilder().setTableId(TABEL_FOR_ARP_FLOW)
 401             .setFlowName(ARP_REPLY_TO_CONTROLLER_FLOW_NAME)
 402             .setPriority(ARP_REPLY_TO_CONTROLLER_FLOW_PRIORITY)
 403             .setBufferId(OFConstants.OFP_NO_BUFFER)
 404             .setIdleTimeout(0)
 405             .setHardTimeout(0)
 406             .setFlags(new FlowModFlags(false, false, false, false, false));
 407         EthernetMatch ethernetMatch = ArpFlowFactory.createEthernetMatch();
 408         ArpMatch arpMatch = ArpFlowFactory.createArpMatch(senderAddress, ipForRequestedMac);
 409         Match match = new MatchBuilder().setEthernetMatch(ethernetMatch).setLayer3Match(arpMatch).setInPort(inPort).build();
 410         arpFlow.setMatch(match);
 411         arpFlow.setInstructions(new InstructionsBuilder().setInstruction(
 412                 ImmutableList.of(SEND_TO_CONTROLLER_INSTRUCTION)).build());
 413         arpFlow.setId(createFlowId(ethernetMatch, arpMatch));
 414         return arpFlow.build();
 415     }
 416
 417     private FlowId createFlowId(EthernetMatch ethernetMatch, ArpMatch arpMatch) {
 418         StringBuilder sb = new StringBuilder();
 419         sb.append(ARP_REPLY_TO_CONTROLLER_FLOW_NAME);
 420         sb.append("|").append(ethernetMatch);
 421         sb.append("|").append(arpMatch);
 422         return new FlowId(sb.toString());
 423     }
 424
 425     private static InstanceIdentifier<Flow> createFlowIid(Flow flow, InstanceIdentifier<Node> nodeIid) {
 426         return nodeIid.builder()
 427             .augmentation(FlowCapableNode.class)
 428             .child(Table.class, new TableKey(flow.getTableId()))
 429             .child(Flow.class, new FlowKey(flow.getId()))
 430             .build();
 431     }
 432
 433     private FutureCallback<RpcResult<Void>> logResult(final Ipv4Address tpa,
 434             final InstanceIdentifier<NodeConnector> ncIid) {
 435         return new FutureCallback<RpcResult<Void>>() {
 436
 437             @Override
 438             public void onSuccess(RpcResult<Void> result) {
 439                 LOG.debug("ARP Request for IP {} was sent from {}.", tpa.getValue(), ncIid);
 440             }
 441
 442             @Override
 443             public void onFailure(Throwable t) {
 444                 LOG.warn("ARP Request for IP {} was NOT sent from {}.", tpa.getValue(), ncIid);
 445             }
 446         };
 447     }
 448
 449     private static String createKey(NodeId node, Ipv4Address ip) {
 450         return node.getValue() + "_" + "_" + ip.getValue();
 451     }
 452
 453 }