renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/arp/ArpTasker.java

   1 /*
   2  * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.arp;
  10
  11 import static com.google.common.base.Preconditions.checkNotNull;
  12
  13 import java.util.ArrayList;
  14 import java.util.List;
  15 import java.util.concurrent.Future;
  16
  17 import javax.annotation.Nonnull;
  18 import javax.annotation.Nullable;
  19
  20 import org.apache.commons.lang3.tuple.ImmutablePair;
  21 import org.apache.commons.lang3.tuple.Pair;
  22 import org.apache.commons.net.util.SubnetUtils;
  23 import org.apache.commons.net.util.SubnetUtils.SubnetInfo;
  24 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  25 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  26 import org.opendaylight.controller.md.sal.binding.api.ReadTransaction;
  27 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  28 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  29 import org.opendaylight.groupbasedpolicy.util.DataStoreHelper;
  30 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  31 import org.opendaylight.openflowplugin.api.OFConstants;
  32 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  33 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  34 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNodeConnector;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowInputBuilder;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowOutput;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.RemoveFlowInput;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.RemoveFlowInputBuilder;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.SalFlowService;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowModFlags;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowRef;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.ApplyActionsCaseBuilder;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActions;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActionsBuilder;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.ContextId;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.L2BridgeDomainId;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.L2ContextId;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.L2FloodDomainId;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.SubnetId;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointKey;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Builder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContextBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayL3Context;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayNodeConfig;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.nodes.node.ExternalInterfaces;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L2BridgeDomain;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L2FloodDomain;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.Subnet;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnector;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnectorKey;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatch;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatch;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketProcessingListener;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketProcessingService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketReceived;
  92 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  93 import org.opendaylight.yangtools.yang.common.RpcResult;
  94 import org.slf4j.Logger;
  95 import org.slf4j.LoggerFactory;
  96
  97 import com.google.common.base.Optional;
  98 import com.google.common.collect.ArrayListMultimap;
  99 import com.google.common.collect.HashMultimap;
 100 import com.google.common.collect.ImmutableList;
 101 import com.google.common.collect.ImmutableSetMultimap;
 102 import com.google.common.collect.ListMultimap;
 103 import com.google.common.collect.Multimaps;
 104 import com.google.common.collect.SetMultimap;
 105 import com.google.common.util.concurrent.FutureCallback;
 106 import com.google.common.util.concurrent.Futures;
 107 import com.google.common.util.concurrent.JdkFutureAdapters;
 108 import com.google.common.util.concurrent.ListenableFuture;
 109 import com.google.common.util.concurrent.MoreExecutors;
 110
 111 public class ArpTasker implements PacketProcessingListener {
 112
 113     private static final Logger LOG = LoggerFactory.getLogger(ArpTasker.class);
 114     private static final short TABEL_FOR_ARP_FLOW = 0;
 115     private static final String ARP_REPLY_TO_CONTROLLER_FLOW_NAME = "arpReplyToController";
 116     private static final int ARP_REPLY_TO_CONTROLLER_FLOW_PRIORITY = 10000;
 117     private static final Instruction SEND_TO_CONTROLLER_INSTRUCTION;
 118     private final ArpSender arpSender;
 119     private final SalFlowService flowService;
 120     private final DataBroker dataProvider;
 121     private final ListMultimap<String, Pair<RemoveFlowInput, EndpointL3Key>> requestInfoByKey = Multimaps.synchronizedListMultimap(ArrayListMultimap.<String, Pair<RemoveFlowInput, EndpointL3Key>>create());
 122
 123     static {
 124         ApplyActions applyActions = new ApplyActionsBuilder().setAction(
 125                 ImmutableList.of(ArpFlowFactory.createSendToControllerAction(0))).build();
 126         SEND_TO_CONTROLLER_INSTRUCTION = new InstructionBuilder().setOrder(0)
 127             .setInstruction(new ApplyActionsCaseBuilder().setApplyActions(applyActions).build())
 128             .build();
 129     }
 130
 131     public ArpTasker(DataBroker dataProvider, PacketProcessingService packetProcessingService,
 132             SalFlowService flowService) {
 133         this.dataProvider = checkNotNull(dataProvider);
 134         this.arpSender = new ArpSender(packetProcessingService);
 135         this.flowService = flowService;
 136     }
 137
 138     @Override
 139     public void onPacketReceived(PacketReceived potentialArp) {
 140         Arp arp = null;
 141         try {
 142             arp = ArpResolverUtils.getArpFrom(potentialArp);
 143         } catch (Exception e) {
 144             LOG.trace(
 145                     "Failed to decode potential ARP packet. This could occur when other than ARP packet was received.",
 146                     e);
 147             return;
 148         }
 149         if (arp.getOperation() != ArpOperation.REPLY.intValue()) {
 150             LOG.trace("ARP packet is not REPLY.");
 151             return;
 152         }
 153         if (LOG.isTraceEnabled()) {
 154             LOG.trace("ARP REPLY received - {}", ArpUtils.getArpToStringFormat(arp));
 155         }
 156         NodeKey nodeKey = potentialArp.getIngress().getValue().firstKeyOf(Node.class, NodeKey.class);
 157         if (nodeKey == null) {
 158             LOG.info("Unknown source node of ARP packet: {}", potentialArp);
 159             return;
 160         }
 161         Ipv4Address spa = ArpUtils.bytesToIp(arp.getSenderProtocolAddress());
 162         MacAddress sha = ArpUtils.bytesToMac(arp.getSenderHardwareAddress());
 163         List<Pair<RemoveFlowInput, EndpointL3Key>> removeFlowInputsAndL3EpKeys = requestInfoByKey.get(createKey(nodeKey.getId(), spa));
 164         for (Pair<RemoveFlowInput, EndpointL3Key> removeFlowInputAndL3EpKey : removeFlowInputsAndL3EpKeys) {
 165             flowService.removeFlow(removeFlowInputAndL3EpKey.getLeft());
 166         }
 167         // each L3EpKey on right-side part of Pair is same
 168         final EndpointL3Key l3EpKey = removeFlowInputsAndL3EpKeys.get(0).getRight();
 169         ReadWriteTransaction rwTx = dataProvider.newReadWriteTransaction();
 170         InstanceIdentifier<EndpointL3> l3EpIid = IidFactory.l3EndpointIid(l3EpKey.getL3Context(),
 171                 l3EpKey.getIpAddress());
 172         Optional<EndpointL3> potentialL3Ep = DataStoreHelper.readFromDs(LogicalDatastoreType.OPERATIONAL, l3EpIid, rwTx);
 173         if (!potentialL3Ep.isPresent()) {
 174             LOG.info("L3 endpoint {} where MAC should be added does not exist anymore.", l3EpKey);
 175             rwTx.cancel();
 176             return;
 177         }
 178         final EndpointL3Builder updatedL3EpBuilder = new EndpointL3Builder(potentialL3Ep.get()).setMacAddress(sha)
 179             .setTimestamp(System.currentTimeMillis());
 180         EndpointL3 updatedL3Ep = updatedL3EpBuilder.build();
 181         L2BridgeDomainId l2BdId = resolveL2BridgeDomainId(updatedL3Ep, rwTx);
 182         if (l2BdId != null) {
 183             updatedL3Ep = updatedL3EpBuilder.setL2Context(l2BdId).build();
 184             EndpointBuilder newEpBuilder = new EndpointBuilder(updatedL3Ep).setKey(new EndpointKey(l2BdId, sha));
 185             OfOverlayL3Context augmentation = updatedL3Ep.getAugmentation(OfOverlayL3Context.class);
 186             if (augmentation != null) {
 187                 newEpBuilder.addAugmentation(OfOverlayContext.class, new OfOverlayContextBuilder(augmentation).build());
 188             }
 189             Endpoint newEp = newEpBuilder.build();
 190             rwTx.put(LogicalDatastoreType.OPERATIONAL, IidFactory.endpointIid(l2BdId, sha), newEp);
 191             LOG.trace("Endpoint was created {}", newEp);
 192         }
 193         rwTx.put(LogicalDatastoreType.OPERATIONAL, l3EpIid, updatedL3Ep);
 194         LOG.trace("MAC was added to L3 endpoint {}", updatedL3Ep);
 195         rwTx.submit();
 196     }
 197
 198     private @Nullable L2BridgeDomainId resolveL2BridgeDomainId(EndpointL3 l3Ep, ReadTransaction rTx) {
 199         TenantId tenantId = l3Ep.getTenant();
 200         Subnet subnetOfL3Ep = readSubnet(l3Ep, rTx);
 201         if (subnetOfL3Ep == null) {
 202             return null;
 203         }
 204         ContextId parentOfSubnet = subnetOfL3Ep.getParent();
 205         if (parentOfSubnet == null) {
 206             return null;
 207         }
 208         L2ContextId l2ContextId = new L2ContextId(parentOfSubnet);
 209         Optional<L2BridgeDomain> potentialL2Bd = DataStoreHelper.readFromDs(LogicalDatastoreType.OPERATIONAL,
 210                 IidFactory.l2BridgeDomainIid(tenantId, new L2BridgeDomainId(l2ContextId)), rTx);
 211         if (potentialL2Bd.isPresent()) {
 212             return potentialL2Bd.get().getId();
 213         }
 214         Optional<L2FloodDomain> potentialL2Fd = DataStoreHelper.readFromDs(LogicalDatastoreType.OPERATIONAL,
 215                 IidFactory.l2FloodDomainIid(tenantId, new L2FloodDomainId(l2ContextId)), rTx);
 216         if (!potentialL2Fd.isPresent()) {
 217             return null;
 218         }
 219         return potentialL2Fd.get().getParent();
 220     }
 221
 222     /**
 223      * Uses ARP to get MAC for the given L3 endpoint. Tries to find MAC for IP from
 224      * {@link EndpointL3#getKey()}.<br>
 225      * {@link EndpointL3#getNetworkContainment()} has to point to a {@link Subnet}.<br>
 226      * ARP Request is sent from all node connectors obtaining from
 227      * {@link OfOverlayNodeConfig#getExternalInterfaces()}<br>
 228      * MAC address obtained from ARP reply is added to the given L3 endpoint (if still exits).<br>
 229      * Also an {@link Endpoint} is created based on MAC If the subnet from network containment point
 230      * to {@link L2BridgeDomain} directly or throught {@link L2FloodDomain}.
 231      *
 232      * @param l3Ep the L3 endpoint which needs to have an MAC address
 233      */
 234     public void addMacForL3EpAndCreateEp(final EndpointL3 l3Ep) {
 235         final Ipv4Address tpa = getIPv4Addresses(l3Ep);
 236         if (tpa == null) {
 237             LOG.debug("L3 endpoint {} does not contain IPv4 address.", l3Ep.getKey());
 238             return;
 239         }
 240         ReadOnlyTransaction rTx = dataProvider.newReadOnlyTransaction();
 241         final SetMultimap<Node, Pair<InstanceIdentifier<NodeConnector>, MacAddress>> extNcWithMacByNode = readNodesWithExternalIfaces(rTx);
 242         if (extNcWithMacByNode.isEmpty()) {
 243             LOG.debug("No node with external interface was found.");
 244             rTx.close();
 245             return;
 246         }
 247         final Ipv4Address senderIpAddress = createSenderIpAddress(l3Ep, rTx);
 248         if (senderIpAddress == null) {
 249             LOG.warn("Cannot create sender IPv4 address for L3 endpoint {}", l3Ep);
 250             rTx.close();
 251             return;
 252         }
 253         rTx.close();
 254
 255         for (final Node node : extNcWithMacByNode.keySet()) {
 256             final InstanceIdentifier<Node> nodeIid = InstanceIdentifier.builder(Nodes.class)
 257                 .child(Node.class, node.getKey())
 258                 .build();
 259             final NodeRef nodeRef = new NodeRef(nodeIid);
 260             List<ListenableFuture<RpcResult<AddFlowOutput>>> arpFlowResultFutures = new ArrayList<>();
 261             List<Pair<RemoveFlowInput, EndpointL3Key>> flowsForRemove = new ArrayList<>();
 262             for (final Pair<InstanceIdentifier<NodeConnector>, MacAddress> extNcIidAndMac : extNcWithMacByNode.get(node)) {
 263                 final ArpMessageAddress senderAddress = new ArpMessageAddress(extNcIidAndMac.getRight(),
 264                         senderIpAddress);
 265                 NodeConnectorId ncId = extNcIidAndMac.getLeft().firstKeyOf(NodeConnector.class, NodeConnectorKey.class).getId();
 266                 final Flow arpReplyToControllerFlow = createArpReplyToControllerFlow(senderAddress, tpa, ncId);
 267                 flowsForRemove.add(new ImmutablePair<>(new RemoveFlowInputBuilder(arpReplyToControllerFlow).setNode(
 268                         nodeRef).build(), l3Ep.getKey()));
 269                 final InstanceIdentifier<Flow> flowIid = createFlowIid(arpReplyToControllerFlow, nodeIid);
 270                 Future<RpcResult<AddFlowOutput>> futureAddFlowResult = flowService.addFlow(new AddFlowInputBuilder(
 271                         arpReplyToControllerFlow).setFlowRef(new FlowRef(flowIid)).setNode(nodeRef).build());
 272                 arpFlowResultFutures.add(JdkFutureAdapters.listenInPoolThread(futureAddFlowResult));
 273             }
 274             requestInfoByKey.putAll(createKey(node.getId(), tpa), flowsForRemove);
 275             ListenableFuture<List<RpcResult<AddFlowOutput>>> futureArpFlowResults = Futures.allAsList(arpFlowResultFutures);
 276             Futures.addCallback(futureArpFlowResults, new FutureCallback<List<RpcResult<AddFlowOutput>>>() {
 277
 278                 @Override
 279                 public void onSuccess(List<RpcResult<AddFlowOutput>> result) {
 280                     for (RpcResult<AddFlowOutput> addFlowResult : result) {
 281                         if (!addFlowResult.isSuccessful()) {
 282                             LOG.warn("An ARP Reply to Controller flow was not created on node {} \nErrors: {}",
 283                                     node.getId().getValue(), addFlowResult.getErrors());
 284                         }
 285                     }
 286                     LOG.debug("ARP Reply to Controller flows were created on node {}", node.getId().getValue());
 287                     for (final Pair<InstanceIdentifier<NodeConnector>, MacAddress> extNcIidAndMac : extNcWithMacByNode.get(node)) {
 288                         final ArpMessageAddress senderAddress = new ArpMessageAddress(extNcIidAndMac.getRight(),
 289                                 senderIpAddress);
 290                         ListenableFuture<RpcResult<Void>> futureSendArpResult = arpSender.sendArp(senderAddress, tpa,
 291                                 extNcIidAndMac.getLeft());
 292                         Futures.addCallback(futureSendArpResult, logResult(tpa, extNcIidAndMac.getLeft()), MoreExecutors.directExecutor());
 293                     }
 294                 }
 295
 296                 @Override
 297                 public void onFailure(@Nonnull Throwable t) {
 298                     LOG.error(
 299                             "Illegal state - Installation of ARP flows on node {} failed. Node can contain just some ARP flows.",
 300                             node.getId(), t);
 301                 }
 302             }, MoreExecutors.directExecutor());
 303         }
 304     }
 305
 306     private static @Nullable Ipv4Address getIPv4Addresses(EndpointL3 l3ep) {
 307         IpAddress ipAddress = l3ep.getKey().getIpAddress();
 308         if (ipAddress.getIpv4Address() == null) {
 309             return null;
 310         }
 311         return ipAddress.getIpv4Address();
 312     }
 313
 314     private SetMultimap<Node, Pair<InstanceIdentifier<NodeConnector>, MacAddress>> readNodesWithExternalIfaces(
 315             ReadTransaction rTx) {
 316         Optional<Nodes> potentialNodes = DataStoreHelper.readFromDs(LogicalDatastoreType.CONFIGURATION,
 317                 InstanceIdentifier.builder(Nodes.class).build(), rTx);
 318         if (!potentialNodes.isPresent() || potentialNodes.get().getNode() == null) {
 319             return ImmutableSetMultimap.of();
 320         }
 321         List<Node> nodes = potentialNodes.get().getNode();
 322         SetMultimap<Node, Pair<InstanceIdentifier<NodeConnector>, MacAddress>> extIfacesByNode = HashMultimap.create();
 323         for (Node node : nodes) {
 324             OfOverlayNodeConfig ofOverlayNode = node.getAugmentation(OfOverlayNodeConfig.class);
 325             if (ofOverlayNode != null) {
 326                 List<ExternalInterfaces> externalIfaces = ofOverlayNode.getExternalInterfaces();
 327                 if (externalIfaces != null) {
 328                     for (ExternalInterfaces extIface : externalIfaces) {
 329                         NodeConnectorId externalNc = extIface.getNodeConnectorId();
 330                         InstanceIdentifier<NodeConnector> extNcIid = InstanceIdentifier.builder(Nodes.class)
 331                             .child(Node.class, node.getKey())
 332                             .child(NodeConnector.class, new NodeConnectorKey(externalNc))
 333                             .build();
 334                         Optional<NodeConnector> potentialExtNcFromOper = DataStoreHelper.readFromDs(
 335                                 LogicalDatastoreType.OPERATIONAL, extNcIid, rTx);
 336                         if (!potentialExtNcFromOper.isPresent()) {
 337                             LOG.debug("Node connector {} does not exit in OPER DS. Node from CONF: {}",
 338                                     externalNc.getValue(), node);
 339                             continue;
 340                         }
 341                         FlowCapableNodeConnector externalFcNc = potentialExtNcFromOper.get().getAugmentation(
 342                                 FlowCapableNodeConnector.class);
 343                         if (externalFcNc == null || externalFcNc.getHardwareAddress() == null) {
 344                             LOG.debug("Hardware address does not exist on node connector {}", externalNc.getValue());
 345                             LOG.trace("Node connector from OPER DS {}", potentialExtNcFromOper.get());
 346                             continue;
 347                         }
 348                         extIfacesByNode.put(node, new ImmutablePair<>(extNcIid, externalFcNc.getHardwareAddress()));
 349                     }
 350                 }
 351             }
 352         }
 353         return extIfacesByNode;
 354     }
 355
 356     private @Nullable Ipv4Address createSenderIpAddress(EndpointL3 l3Ep, ReadTransaction rTx) {
 357         Subnet subnetOfL3Ep = readSubnet(l3Ep, rTx);
 358         if (subnetOfL3Ep == null) {
 359             return null;
 360         }
 361         SubnetInfo subnetInfo = new SubnetUtils(subnetOfL3Ep.getIpPrefix().getIpv4Prefix().getValue()).getInfo();
 362         String senderIp = subnetInfo.getHighAddress();
 363         if (senderIp.equals(l3Ep.getKey().getIpAddress().getIpv4Address().getValue())) {
 364             senderIp = subnetInfo.getLowAddress();
 365         }
 366         return new Ipv4Address(senderIp);
 367     }
 368
 369     private @Nullable Subnet readSubnet(EndpointL3 l3Ep, ReadTransaction rTx) {
 370         NetworkDomainId l3EpNetworkContainment = l3Ep.getNetworkContainment();
 371         if (l3EpNetworkContainment == null) {
 372             LOG.debug("L3 endpoint {} does not contain network containment.", l3Ep.getKey());
 373             return null;
 374         }
 375         if (l3Ep.getTenant() == null) {
 376             LOG.debug("L3 endpoint {} does not contain tenat.", l3Ep.getKey());
 377             return null;
 378         }
 379         Optional<Subnet> potentialSubnet = DataStoreHelper.readFromDs(LogicalDatastoreType.CONFIGURATION,
 380                 IidFactory.subnetIid(l3Ep.getTenant(), new SubnetId(l3EpNetworkContainment)), rTx);
 381         if (!potentialSubnet.isPresent()) {
 382             LOG.debug(
 383                     "Network containment {} of L3 endpoint {} does not point to a subnet or the subnet does not exist.",
 384                     l3EpNetworkContainment.getValue(), l3Ep.getKey());
 385             return null;
 386         }
 387         return potentialSubnet.get();
 388     }
 389
 390     private Flow createArpReplyToControllerFlow(ArpMessageAddress senderAddress, Ipv4Address ipForRequestedMac, NodeConnectorId inPort) {
 391         checkNotNull(senderAddress);
 392         checkNotNull(ipForRequestedMac);
 393         FlowBuilder arpFlow = new FlowBuilder().setTableId(TABEL_FOR_ARP_FLOW)
 394             .setFlowName(ARP_REPLY_TO_CONTROLLER_FLOW_NAME)
 395             .setPriority(ARP_REPLY_TO_CONTROLLER_FLOW_PRIORITY)
 396             .setBufferId(OFConstants.OFP_NO_BUFFER)
 397             .setIdleTimeout(0)
 398             .setHardTimeout(0)
 399             .setFlags(new FlowModFlags(false, false, false, false, false));
 400         EthernetMatch ethernetMatch = ArpFlowFactory.createEthernetMatch();
 401         ArpMatch arpMatch = ArpFlowFactory.createArpMatch(senderAddress, ipForRequestedMac);
 402         Match match = new MatchBuilder().setEthernetMatch(ethernetMatch).setLayer3Match(arpMatch).setInPort(inPort).build();
 403         arpFlow.setMatch(match);
 404         arpFlow.setInstructions(new InstructionsBuilder().setInstruction(
 405                 ImmutableList.of(SEND_TO_CONTROLLER_INSTRUCTION)).build());
 406         arpFlow.setId(createFlowId(ethernetMatch, arpMatch));
 407         return arpFlow.build();
 408     }
 409
 410     private FlowId createFlowId(EthernetMatch ethernetMatch, ArpMatch arpMatch) {
 411         StringBuilder sb = new StringBuilder();
 412         sb.append(ARP_REPLY_TO_CONTROLLER_FLOW_NAME);
 413         sb.append("|").append(ethernetMatch);
 414         sb.append("|").append(arpMatch);
 415         return new FlowId(sb.toString());
 416     }
 417
 418     private static InstanceIdentifier<Flow> createFlowIid(Flow flow, InstanceIdentifier<Node> nodeIid) {
 419         return nodeIid.builder()
 420             .augmentation(FlowCapableNode.class)
 421             .child(Table.class, new TableKey(flow.getTableId()))
 422             .child(Flow.class, new FlowKey(flow.getId()))
 423             .build();
 424     }
 425
 426     private FutureCallback<RpcResult<Void>> logResult(final Ipv4Address tpa,
 427             final InstanceIdentifier<NodeConnector> ncIid) {
 428         return new FutureCallback<RpcResult<Void>>() {
 429
 430             @Override
 431             public void onSuccess(RpcResult<Void> result) {
 432                 LOG.debug("ARP Request for IP {} was sent from {}.", tpa.getValue(), ncIid);
 433             }
 434
 435             @Override
 436             public void onFailure(Throwable t) {
 437                 LOG.warn("ARP Request for IP {} was NOT sent from {}.", tpa.getValue(), ncIid);
 438             }
 439         };
 440     }
 441
 442     private static String createKey(NodeId node, Ipv4Address ip) {
 443         return node.getValue() + "_" + "_" + ip.getValue();
 444     }
 445
 446 }