renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/arp/ArpTasker.java

   1 /*
   2  * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.arp;
  10
  11 import static com.google.common.base.Preconditions.checkNotNull;
  12
  13 import java.util.ArrayList;
  14 import java.util.List;
  15 import java.util.concurrent.Future;
  16
  17 import javax.annotation.Nullable;
  18
  19 import org.apache.commons.lang3.tuple.ImmutablePair;
  20 import org.apache.commons.lang3.tuple.Pair;
  21 import org.apache.commons.net.util.SubnetUtils;
  22 import org.apache.commons.net.util.SubnetUtils.SubnetInfo;
  23 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  24 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  25 import org.opendaylight.controller.md.sal.binding.api.ReadTransaction;
  26 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  28 import org.opendaylight.groupbasedpolicy.util.DataStoreHelper;
  29 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  30 import org.opendaylight.openflowplugin.api.OFConstants;
  31 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  32 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  33 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNodeConnector;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowInputBuilder;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowOutput;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.RemoveFlowInput;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.RemoveFlowInputBuilder;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.SalFlowService;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowModFlags;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowRef;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.ApplyActionsCaseBuilder;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActions;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActionsBuilder;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.ContextId;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.L2BridgeDomainId;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.L2ContextId;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.L2FloodDomainId;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.SubnetId;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointKey;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Builder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContextBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayL3Context;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayNodeConfig;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.nodes.node.ExternalInterfaces;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L2BridgeDomain;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L2FloodDomain;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.Subnet;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnector;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnectorKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatch;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatch;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketProcessingListener;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketProcessingService;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketReceived;
  91 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  92 import org.opendaylight.yangtools.yang.common.RpcResult;
  93 import org.slf4j.Logger;
  94 import org.slf4j.LoggerFactory;
  95
  96 import com.google.common.base.Optional;
  97 import com.google.common.collect.ArrayListMultimap;
  98 import com.google.common.collect.HashMultimap;
  99 import com.google.common.collect.ImmutableList;
 100 import com.google.common.collect.ImmutableSetMultimap;
 101 import com.google.common.collect.ListMultimap;
 102 import com.google.common.collect.Multimaps;
 103 import com.google.common.collect.SetMultimap;
 104 import com.google.common.util.concurrent.FutureCallback;
 105 import com.google.common.util.concurrent.Futures;
 106 import com.google.common.util.concurrent.JdkFutureAdapters;
 107 import com.google.common.util.concurrent.ListenableFuture;
 108
 109 public class ArpTasker implements PacketProcessingListener {
 110
 111     private static final Logger LOG = LoggerFactory.getLogger(ArpTasker.class);
 112     private static final short TABEL_FOR_ARP_FLOW = 0;
 113     private static final String ARP_REPLY_TO_CONTROLLER_FLOW_NAME = "arpReplyToController";
 114     private static final int ARP_REPLY_TO_CONTROLLER_FLOW_PRIORITY = 10000;
 115     private static final Instruction SEND_TO_CONTROLLER_INSTRUCTION;
 116     private final ArpSender arpSender;
 117     private final SalFlowService flowService;
 118     private final DataBroker dataProvider;
 119     private final ListMultimap<String, Pair<RemoveFlowInput, EndpointL3Key>> requestInfoByKey = Multimaps.synchronizedListMultimap(ArrayListMultimap.<String, Pair<RemoveFlowInput, EndpointL3Key>>create());
 120
 121     static {
 122         ApplyActions applyActions = new ApplyActionsBuilder().setAction(
 123                 ImmutableList.of(ArpFlowFactory.createSendToControllerAction(0))).build();
 124         SEND_TO_CONTROLLER_INSTRUCTION = new InstructionBuilder().setOrder(0)
 125             .setInstruction(new ApplyActionsCaseBuilder().setApplyActions(applyActions).build())
 126             .build();
 127     }
 128
 129     public ArpTasker(DataBroker dataProvider, PacketProcessingService packetProcessingService,
 130             SalFlowService flowService) {
 131         this.dataProvider = checkNotNull(dataProvider);
 132         this.arpSender = new ArpSender(packetProcessingService);
 133         this.flowService = flowService;
 134     }
 135
 136     @Override
 137     public void onPacketReceived(PacketReceived potentialArp) {
 138         Arp arp = null;
 139         try {
 140             arp = ArpResolverUtils.getArpFrom(potentialArp);
 141         } catch (Exception e) {
 142             LOG.trace(
 143                     "Failed to decode potential ARP packet. This could occur when other than ARP packet was received.",
 144                     e);
 145             return;
 146         }
 147         if (arp.getOperation() != ArpOperation.REPLY.intValue()) {
 148             LOG.trace("ARP packet is not REPLY.");
 149             return;
 150         }
 151         if (LOG.isTraceEnabled()) {
 152             LOG.trace("ARP REPLY received - {}", ArpUtils.getArpToStringFormat(arp));
 153         }
 154         NodeKey nodeKey = potentialArp.getIngress().getValue().firstKeyOf(Node.class, NodeKey.class);
 155         if (nodeKey == null) {
 156             LOG.info("Unknown source node of ARP packet: {}", potentialArp);
 157             return;
 158         }
 159         Ipv4Address spa = ArpUtils.bytesToIp(arp.getSenderProtocolAddress());
 160         MacAddress sha = ArpUtils.bytesToMac(arp.getSenderHardwareAddress());
 161         List<Pair<RemoveFlowInput, EndpointL3Key>> removeFlowInputsAndL3EpKeys = requestInfoByKey.get(createKey(nodeKey.getId(), spa));
 162         for (Pair<RemoveFlowInput, EndpointL3Key> removeFlowInputAndL3EpKey : removeFlowInputsAndL3EpKeys) {
 163             flowService.removeFlow(removeFlowInputAndL3EpKey.getLeft());
 164         }
 165         // each L3EpKey on right-side part of Pair is same
 166         final EndpointL3Key l3EpKey = removeFlowInputsAndL3EpKeys.get(0).getRight();
 167         ReadWriteTransaction rwTx = dataProvider.newReadWriteTransaction();
 168         InstanceIdentifier<EndpointL3> l3EpIid = IidFactory.l3EndpointIid(l3EpKey.getL3Context(),
 169                 l3EpKey.getIpAddress());
 170         Optional<EndpointL3> potentialL3Ep = DataStoreHelper.readFromDs(LogicalDatastoreType.OPERATIONAL, l3EpIid, rwTx);
 171         if (!potentialL3Ep.isPresent()) {
 172             LOG.info("L3 endpoint {} where MAC should be added does not exist anymore.", l3EpKey);
 173             rwTx.cancel();
 174             return;
 175         }
 176         final EndpointL3Builder updatedL3EpBuilder = new EndpointL3Builder(potentialL3Ep.get()).setMacAddress(sha)
 177             .setTimestamp(System.currentTimeMillis());
 178         EndpointL3 updatedL3Ep = updatedL3EpBuilder.build();
 179         L2BridgeDomainId l2BdId = resolveL2BridgeDomainId(updatedL3Ep, rwTx);
 180         if (l2BdId != null) {
 181             updatedL3Ep = updatedL3EpBuilder.setL2Context(l2BdId).build();
 182             EndpointBuilder newEpBuilder = new EndpointBuilder(updatedL3Ep).setKey(new EndpointKey(l2BdId, sha));
 183             OfOverlayL3Context augmentation = updatedL3Ep.getAugmentation(OfOverlayL3Context.class);
 184             if (augmentation != null) {
 185                 newEpBuilder.addAugmentation(OfOverlayContext.class, new OfOverlayContextBuilder(augmentation).build());
 186             }
 187             Endpoint newEp = newEpBuilder.build();
 188             rwTx.put(LogicalDatastoreType.OPERATIONAL, IidFactory.endpointIid(l2BdId, sha), newEp);
 189             LOG.trace("Endpoint was created {}", newEp);
 190         }
 191         rwTx.put(LogicalDatastoreType.OPERATIONAL, l3EpIid, updatedL3Ep);
 192         LOG.trace("MAC was added to L3 endpoint {}", updatedL3Ep);
 193         rwTx.submit();
 194     }
 195
 196     private @Nullable L2BridgeDomainId resolveL2BridgeDomainId(EndpointL3 l3Ep, ReadTransaction rTx) {
 197         TenantId tenantId = l3Ep.getTenant();
 198         Subnet subnetOfL3Ep = readSubnet(l3Ep, rTx);
 199         if (subnetOfL3Ep == null) {
 200             return null;
 201         }
 202         ContextId parentOfSubnet = subnetOfL3Ep.getParent();
 203         if (parentOfSubnet == null) {
 204             return null;
 205         }
 206         L2ContextId l2ContextId = new L2ContextId(parentOfSubnet);
 207         Optional<L2BridgeDomain> potentialL2Bd = DataStoreHelper.readFromDs(LogicalDatastoreType.OPERATIONAL,
 208                 IidFactory.l2BridgeDomainIid(tenantId, new L2BridgeDomainId(l2ContextId)), rTx);
 209         if (potentialL2Bd.isPresent()) {
 210             return potentialL2Bd.get().getId();
 211         }
 212         Optional<L2FloodDomain> potentialL2Fd = DataStoreHelper.readFromDs(LogicalDatastoreType.OPERATIONAL,
 213                 IidFactory.l2FloodDomainIid(tenantId, new L2FloodDomainId(l2ContextId)), rTx);
 214         if (!potentialL2Fd.isPresent()) {
 215             return null;
 216         }
 217         return potentialL2Fd.get().getParent();
 218     }
 219
 220     /**
 221      * Uses ARP to get MAC for the given L3 endpoint. Tries to find MAC for IP from
 222      * {@link EndpointL3#getKey()}.<br>
 223      * {@link EndpointL3#getNetworkContainment()} has to point to a {@link Subnet}.<br>
 224      * ARP Request is sent from all node connectors obtaining from
 225      * {@link OfOverlayNodeConfig#getExternalInterfaces()}<br>
 226      * MAC address obtained from ARP reply is added to the given L3 endpoint (if still exits).<br>
 227      * Also an {@link Endpoint} is created based on MAC If the subnet from network containment point
 228      * to {@link L2BridgeDomain} directly or throught {@link L2FloodDomain}.
 229      *
 230      * @param l3Ep the L3 endpoint which needs to have an MAC address
 231      */
 232     public void addMacForL3EpAndCreateEp(final EndpointL3 l3Ep) {
 233         final Ipv4Address tpa = getIPv4Addresses(l3Ep);
 234         if (tpa == null) {
 235             LOG.debug("L3 endpoint {} does not contain IPv4 address.", l3Ep.getKey());
 236             return;
 237         }
 238         ReadOnlyTransaction rTx = dataProvider.newReadOnlyTransaction();
 239         final SetMultimap<Node, Pair<InstanceIdentifier<NodeConnector>, MacAddress>> extNcWithMacByNode = readNodesWithExternalIfaces(rTx);
 240         if (extNcWithMacByNode.isEmpty()) {
 241             LOG.debug("No node with external interface was found.");
 242             rTx.close();
 243             return;
 244         }
 245         final Ipv4Address senderIpAddress = createSenderIpAddress(l3Ep, rTx);
 246         if (senderIpAddress == null) {
 247             LOG.warn("Cannot create sender IPv4 address for L3 endpoint {}", l3Ep);
 248             rTx.close();
 249             return;
 250         }
 251         rTx.close();
 252
 253         for (final Node node : extNcWithMacByNode.keySet()) {
 254             final InstanceIdentifier<Node> nodeIid = InstanceIdentifier.builder(Nodes.class)
 255                 .child(Node.class, node.getKey())
 256                 .build();
 257             final NodeRef nodeRef = new NodeRef(nodeIid);
 258             List<ListenableFuture<RpcResult<AddFlowOutput>>> arpFlowResultFutures = new ArrayList<>();
 259             List<Pair<RemoveFlowInput, EndpointL3Key>> flowsForRemove = new ArrayList<>();
 260             for (final Pair<InstanceIdentifier<NodeConnector>, MacAddress> extNcIidAndMac : extNcWithMacByNode.get(node)) {
 261                 final ArpMessageAddress senderAddress = new ArpMessageAddress(extNcIidAndMac.getRight(),
 262                         senderIpAddress);
 263                 NodeConnectorId ncId = extNcIidAndMac.getLeft().firstKeyOf(NodeConnector.class, NodeConnectorKey.class).getId();
 264                 final Flow arpReplyToControllerFlow = createArpReplyToControllerFlow(senderAddress, tpa, ncId);
 265                 flowsForRemove.add(new ImmutablePair<>(new RemoveFlowInputBuilder(arpReplyToControllerFlow).setNode(
 266                         nodeRef).build(), l3Ep.getKey()));
 267                 final InstanceIdentifier<Flow> flowIid = createFlowIid(arpReplyToControllerFlow, nodeIid);
 268                 Future<RpcResult<AddFlowOutput>> futureAddFlowResult = flowService.addFlow(new AddFlowInputBuilder(
 269                         arpReplyToControllerFlow).setFlowRef(new FlowRef(flowIid)).setNode(nodeRef).build());
 270                 arpFlowResultFutures.add(JdkFutureAdapters.listenInPoolThread(futureAddFlowResult));
 271             }
 272             requestInfoByKey.putAll(createKey(node.getId(), tpa), flowsForRemove);
 273             ListenableFuture<List<RpcResult<AddFlowOutput>>> futureArpFlowResults = Futures.allAsList(arpFlowResultFutures);
 274             Futures.addCallback(futureArpFlowResults, new FutureCallback<List<RpcResult<AddFlowOutput>>>() {
 275
 276                 @Override
 277                 public void onSuccess(List<RpcResult<AddFlowOutput>> result) {
 278                     for (RpcResult<AddFlowOutput> addFlowResult : result) {
 279                         if (!addFlowResult.isSuccessful()) {
 280                             LOG.warn("An ARP Reply to Controller flow was not created on node {} \nErrors: {}",
 281                                     node.getId().getValue(), addFlowResult.getErrors());
 282                             continue;
 283                         }
 284                     }
 285                     LOG.debug("ARP Reply to Controller flows were created on node {}", node.getId().getValue());
 286                     for (final Pair<InstanceIdentifier<NodeConnector>, MacAddress> extNcIidAndMac : extNcWithMacByNode.get(node)) {
 287                         final ArpMessageAddress senderAddress = new ArpMessageAddress(extNcIidAndMac.getRight(),
 288                                 senderIpAddress);
 289                         ListenableFuture<RpcResult<Void>> futureSendArpResult = arpSender.sendArp(senderAddress, tpa,
 290                                 extNcIidAndMac.getLeft());
 291                         Futures.addCallback(futureSendArpResult, logResult(tpa, extNcIidAndMac.getLeft()));
 292                     }
 293                 }
 294
 295                 @Override
 296                 public void onFailure(Throwable t) {
 297                     LOG.error(
 298                             "Illegal state - Installation of ARP flows on node {} failed. Node can contain just some ARP flows.",
 299                             node.getId(), t);
 300                 }
 301             });
 302         }
 303     }
 304
 305     private static @Nullable Ipv4Address getIPv4Addresses(EndpointL3 l3ep) {
 306         IpAddress ipAddress = l3ep.getKey().getIpAddress();
 307         if (ipAddress.getIpv4Address() == null) {
 308             return null;
 309         }
 310         return ipAddress.getIpv4Address();
 311     }
 312
 313     private SetMultimap<Node, Pair<InstanceIdentifier<NodeConnector>, MacAddress>> readNodesWithExternalIfaces(
 314             ReadTransaction rTx) {
 315         Optional<Nodes> potentialNodes = DataStoreHelper.readFromDs(LogicalDatastoreType.CONFIGURATION,
 316                 InstanceIdentifier.builder(Nodes.class).build(), rTx);
 317         if (!potentialNodes.isPresent() || potentialNodes.get().getNode() == null) {
 318             return ImmutableSetMultimap.of();
 319         }
 320         List<Node> nodes = potentialNodes.get().getNode();
 321         SetMultimap<Node, Pair<InstanceIdentifier<NodeConnector>, MacAddress>> extIfacesByNode = HashMultimap.create();
 322         for (Node node : nodes) {
 323             OfOverlayNodeConfig ofOverlayNode = node.getAugmentation(OfOverlayNodeConfig.class);
 324             if (ofOverlayNode != null) {
 325                 List<ExternalInterfaces> externalIfaces = ofOverlayNode.getExternalInterfaces();
 326                 if (externalIfaces != null) {
 327                     for (ExternalInterfaces extIface : externalIfaces) {
 328                         NodeConnectorId externalNc = extIface.getNodeConnectorId();
 329                         InstanceIdentifier<NodeConnector> extNcIid = InstanceIdentifier.builder(Nodes.class)
 330                             .child(Node.class, node.getKey())
 331                             .child(NodeConnector.class, new NodeConnectorKey(externalNc))
 332                             .build();
 333                         Optional<NodeConnector> potentialExtNcFromOper = DataStoreHelper.readFromDs(
 334                                 LogicalDatastoreType.OPERATIONAL, extNcIid, rTx);
 335                         if (!potentialExtNcFromOper.isPresent()) {
 336                             LOG.debug("Node connector {} does not exit in OPER DS. Node from CONF: {}",
 337                                     externalNc.getValue(), node);
 338                             continue;
 339                         }
 340                         FlowCapableNodeConnector externalFcNc = potentialExtNcFromOper.get().getAugmentation(
 341                                 FlowCapableNodeConnector.class);
 342                         if (externalFcNc == null || externalFcNc.getHardwareAddress() == null) {
 343                             LOG.debug("Hardware address does not exist on node connector {}", externalNc.getValue());
 344                             LOG.trace("Node connector from OPER DS {}", potentialExtNcFromOper.get());
 345                             continue;
 346                         }
 347                         extIfacesByNode.put(node, new ImmutablePair<>(extNcIid, externalFcNc.getHardwareAddress()));
 348                     }
 349                 }
 350             }
 351         }
 352         return extIfacesByNode;
 353     }
 354
 355     private @Nullable Ipv4Address createSenderIpAddress(EndpointL3 l3Ep, ReadTransaction rTx) {
 356         Subnet subnetOfL3Ep = readSubnet(l3Ep, rTx);
 357         if (subnetOfL3Ep == null) {
 358             return null;
 359         }
 360         SubnetInfo subnetInfo = new SubnetUtils(subnetOfL3Ep.getIpPrefix().getIpv4Prefix().getValue()).getInfo();
 361         String senderIp = subnetInfo.getHighAddress();
 362         if (senderIp.equals(l3Ep.getKey().getIpAddress().getIpv4Address().getValue())) {
 363             senderIp = subnetInfo.getLowAddress();
 364         }
 365         return new Ipv4Address(senderIp);
 366     }
 367
 368     private @Nullable Subnet readSubnet(EndpointL3 l3Ep, ReadTransaction rTx) {
 369         NetworkDomainId l3EpNetworkContainment = l3Ep.getNetworkContainment();
 370         if (l3EpNetworkContainment == null) {
 371             LOG.debug("L3 endpoint {} does not contain network containment.", l3Ep.getKey());
 372             return null;
 373         }
 374         if (l3Ep.getTenant() == null) {
 375             LOG.debug("L3 endpoint {} does not contain tenat.", l3Ep.getKey());
 376             return null;
 377         }
 378         Optional<Subnet> potentialSubnet = DataStoreHelper.readFromDs(LogicalDatastoreType.CONFIGURATION,
 379                 IidFactory.subnetIid(l3Ep.getTenant(), new SubnetId(l3EpNetworkContainment)), rTx);
 380         if (!potentialSubnet.isPresent()) {
 381             LOG.debug(
 382                     "Network containment {} of L3 endpoint {} does not point to a subnet or the subnet does not exist.",
 383                     l3EpNetworkContainment.getValue(), l3Ep.getKey());
 384             return null;
 385         }
 386         return potentialSubnet.get();
 387     }
 388
 389     private Flow createArpReplyToControllerFlow(ArpMessageAddress senderAddress, Ipv4Address ipForRequestedMac, NodeConnectorId inPort) {
 390         checkNotNull(senderAddress);
 391         checkNotNull(ipForRequestedMac);
 392         FlowBuilder arpFlow = new FlowBuilder().setTableId(TABEL_FOR_ARP_FLOW)
 393             .setFlowName(ARP_REPLY_TO_CONTROLLER_FLOW_NAME)
 394             .setPriority(ARP_REPLY_TO_CONTROLLER_FLOW_PRIORITY)
 395             .setBufferId(OFConstants.OFP_NO_BUFFER)
 396             .setIdleTimeout(0)
 397             .setHardTimeout(0)
 398             .setFlags(new FlowModFlags(false, false, false, false, false));
 399         EthernetMatch ethernetMatch = ArpFlowFactory.createEthernetMatch();
 400         ArpMatch arpMatch = ArpFlowFactory.createArpMatch(senderAddress, ipForRequestedMac);
 401         Match match = new MatchBuilder().setEthernetMatch(ethernetMatch).setLayer3Match(arpMatch).setInPort(inPort).build();
 402         arpFlow.setMatch(match);
 403         arpFlow.setInstructions(new InstructionsBuilder().setInstruction(
 404                 ImmutableList.of(SEND_TO_CONTROLLER_INSTRUCTION)).build());
 405         arpFlow.setId(createFlowId(ethernetMatch, arpMatch));
 406         return arpFlow.build();
 407     }
 408
 409     private FlowId createFlowId(EthernetMatch ethernetMatch, ArpMatch arpMatch) {
 410         StringBuilder sb = new StringBuilder();
 411         sb.append(ARP_REPLY_TO_CONTROLLER_FLOW_NAME);
 412         sb.append("|").append(ethernetMatch);
 413         sb.append("|").append(arpMatch);
 414         return new FlowId(sb.toString());
 415     }
 416
 417     private static InstanceIdentifier<Flow> createFlowIid(Flow flow, InstanceIdentifier<Node> nodeIid) {
 418         return nodeIid.builder()
 419             .augmentation(FlowCapableNode.class)
 420             .child(Table.class, new TableKey(flow.getTableId()))
 421             .child(Flow.class, new FlowKey(flow.getId()))
 422             .build();
 423     }
 424
 425     private FutureCallback<RpcResult<Void>> logResult(final Ipv4Address tpa,
 426             final InstanceIdentifier<NodeConnector> ncIid) {
 427         return new FutureCallback<RpcResult<Void>>() {
 428
 429             @Override
 430             public void onSuccess(RpcResult<Void> result) {
 431                 LOG.debug("ARP Request for IP {} was sent from {}.", tpa.getValue(), ncIid);
 432             }
 433
 434             @Override
 435             public void onFailure(Throwable t) {
 436                 LOG.warn("ARP Request for IP {} was NOT sent from {}.", tpa.getValue(), ncIid);
 437             }
 438         };
 439     }
 440
 441     private static String createKey(NodeId node, Ipv4Address ip) {
 442         return node.getValue() + "_" + "_" + ip.getValue();
 443     }
 444
 445 }