2 * Copyright (c) 2015 Brocade Communications Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.services.arp;
10 import static com.google.common.base.Preconditions.checkNotNull;
12 import java.math.BigInteger;
13 import java.util.Map.Entry;
14 import java.util.concurrent.Callable;
15 import java.util.concurrent.ConcurrentHashMap;
16 import java.util.concurrent.ConcurrentMap;
17 import java.util.concurrent.Executors;
18 import java.util.concurrent.Future;
19 import java.util.concurrent.ScheduledExecutorService;
20 import java.util.concurrent.TimeUnit;
21 import java.util.concurrent.atomic.AtomicBoolean;
22 import java.util.concurrent.atomic.AtomicLong;
24 import org.opendaylight.controller.sal.binding.api.BindingAwareBroker.ProviderContext;
25 import org.opendaylight.openflowplugin.api.OFConstants;
26 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolver;
27 import org.opendaylight.ovsdb.openstack.netvirt.providers.ConfigInterface;
28 import org.opendaylight.ovsdb.openstack.netvirt.providers.NetvirtProvidersProvider;
29 import org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.AbstractServiceInstance;
30 import org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.Service;
31 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
32 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
33 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowInputBuilder;
41 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.AddFlowOutput;
42 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.RemoveFlowInputBuilder;
43 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.SalFlowService;
44 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowCookie;
45 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowModFlags;
46 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.FlowRef;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.ApplyActionsCaseBuilder;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActions;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActionsBuilder;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnector;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnectorKey;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatch;
62 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatch;
63 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketProcessingListener;
64 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketProcessingService;
65 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketReceived;
66 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
67 import org.opendaylight.yangtools.yang.binding.KeyedInstanceIdentifier;
68 import org.opendaylight.yangtools.yang.common.RpcResult;
69 import org.osgi.framework.BundleContext;
70 import org.osgi.framework.ServiceReference;
71 import org.slf4j.Logger;
72 import org.slf4j.LoggerFactory;
74 import com.google.common.base.Preconditions;
75 import com.google.common.collect.ImmutableList;
76 import com.google.common.util.concurrent.FutureCallback;
77 import com.google.common.util.concurrent.Futures;
78 import com.google.common.util.concurrent.JdkFutureAdapters;
79 import com.google.common.util.concurrent.ListenableFuture;
80 import com.google.common.util.concurrent.ListeningExecutorService;
81 import com.google.common.util.concurrent.MoreExecutors;
85 * @author Anil Vishnoi (avishnoi@Brocade.com)
88 public class GatewayMacResolverService extends AbstractServiceInstance
89 implements ConfigInterface, GatewayMacResolver,PacketProcessingListener {
91 private static final Logger LOG = LoggerFactory.getLogger(GatewayMacResolverService.class);
92 private static final short TABLE_FOR_ARP_FLOW = 0;
93 private static final String ARP_REPLY_TO_CONTROLLER_FLOW_NAME = "GatewayArpReplyRouter";
94 private static final int ARP_REPLY_TO_CONTROLLER_FLOW_PRIORITY = 10000;
95 private static final Instruction SEND_TO_CONTROLLER_INSTRUCTION;
96 private ArpSender arpSender;
97 private SalFlowService flowService;
98 private final AtomicLong flowCookie = new AtomicLong();
99 private final ConcurrentMap<Ipv4Address, ArpResolverMetadata> gatewayToArpMetadataMap =
100 new ConcurrentHashMap<>();
101 private static final int ARP_WATCH_BROTHERS = 10;
102 private static final int WAIT_CYCLES = 3;
103 private static final int PER_CYCLE_WAIT_DURATION = 1000;
104 private static final int REFRESH_INTERVAL = 10;
105 private final ListeningExecutorService arpWatcherWall = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(ARP_WATCH_BROTHERS));
106 private final ScheduledExecutorService gatewayMacRefresherPool = Executors.newScheduledThreadPool(1);
107 private final ScheduledExecutorService refreshRequester = Executors.newSingleThreadScheduledExecutor();
108 private AtomicBoolean initializationDone = new AtomicBoolean(false);
111 ApplyActions applyActions = new ApplyActionsBuilder().setAction(
112 ImmutableList.of(ArpFlowFactory.createSendToControllerAction(0))).build();
113 SEND_TO_CONTROLLER_INSTRUCTION = new InstructionBuilder().setOrder(0)
114 .setInstruction(new ApplyActionsCaseBuilder().setApplyActions(applyActions).build())
118 public GatewayMacResolverService(){
119 super(Service.GATEWAY_RESOLVER);
122 public GatewayMacResolverService(Service service){
127 if(!initializationDone.get()){
128 initializationDone.set(true);
129 ProviderContext providerContext = NetvirtProvidersProvider.getProviderContext();
130 checkNotNull(providerContext);
131 PacketProcessingService packetProcessingService = providerContext.getRpcService(PacketProcessingService.class);
132 if (packetProcessingService != null) {
133 LOG.debug("{} was found.", PacketProcessingService.class.getSimpleName());
134 this.arpSender = new ArpSender(packetProcessingService);
136 LOG.error("Missing service {}", PacketProcessingService.class.getSimpleName());
137 this.arpSender = null;
139 flowService = providerContext.getRpcService(SalFlowService.class);
140 refreshRequester.scheduleWithFixedDelay(new Runnable(){
144 if (!gatewayToArpMetadataMap.isEmpty()){
145 for(final Entry<Ipv4Address, ArpResolverMetadata> gatewayToArpMetadataEntry : gatewayToArpMetadataMap.entrySet()){
146 final Ipv4Address gatewayIp = gatewayToArpMetadataEntry.getKey();
147 final ArpResolverMetadata gatewayMetaData = gatewayToArpMetadataEntry.getValue();
148 gatewayMacRefresherPool.schedule(new Runnable(){
153 final Node externalNetworkBridge = getExternalBridge(gatewayMetaData.getExternalNetworkBridgeDpid());
154 if(externalNetworkBridge == null){
155 LOG.error("MAC address for gateway {} can not be resolved, because external bridge {} "
156 + "is not connected to controller.",gatewayIp.getValue(),gatewayMetaData.getExternalNetworkBridgeDpid() );
159 LOG.debug("Refresh Gateway Mac for gateway {} using source ip {} and mac {} for ARP request",
160 gatewayIp.getValue(),gatewayMetaData.getArpRequestSourceIp().getValue(),gatewayMetaData.getArpRequestSourceMacAddress().getValue());
162 sendGatewayArpRequest(externalNetworkBridge,gatewayIp,gatewayMetaData.getArpRequestSourceIp(), gatewayMetaData.getArpRequestSourceMacAddress());
164 }, 1, TimeUnit.SECONDS);
168 }, REFRESH_INTERVAL, REFRESH_INTERVAL, TimeUnit.SECONDS);
172 * Method do following actions:
173 * 1. Install flow to direct ARP response packet to controller
174 * 2. Send ARP request packet out on all port of the given External network bridge.
175 * 3. Cache the flow that need to be removed once ARP resolution is done.
176 * 4. Return listenable future so that user can add callback to get the MacAddress
177 * @param externalNetworkBridgeDpid Broadcast ARP request packet on this bridge
178 * @param gatewayIp IP address for which MAC need to be resolved
179 * @param sourceIpAddress Source Ip address for the ARP request packet
180 * @param sourceMacAddress Source Mac address for the ARP request packet
181 * @param periodicRefresh Enable/Disable periodic refresh of the Gateway Mac address
182 * NOTE:Periodic refresh is not supported yet.
183 * @return Future object
186 public ListenableFuture<MacAddress> resolveMacAddress( final Long externalNetworkBridgeDpid, final Ipv4Address gatewayIp,
187 final Ipv4Address sourceIpAddress, final MacAddress sourceMacAddress, final Boolean periodicRefresh){
188 Preconditions.checkNotNull(sourceIpAddress);
189 Preconditions.checkNotNull(sourceMacAddress);
190 Preconditions.checkNotNull(gatewayIp);
192 LOG.info("Trigger Mac resolution for gateway {}, using source ip {} and mac {}",
193 gatewayIp.getValue(),sourceIpAddress.getValue(),sourceMacAddress.getValue());
196 if(gatewayToArpMetadataMap.containsKey(gatewayIp)){
197 if(gatewayToArpMetadataMap.get(gatewayIp).getGatewayMacAddress() != null){
198 return arpWatcherWall.submit(new Callable<MacAddress>(){
201 public MacAddress call() throws Exception {
202 return gatewayToArpMetadataMap.get(gatewayIp).getGatewayMacAddress();
207 gatewayToArpMetadataMap.put(gatewayIp,new ArpResolverMetadata(
208 externalNetworkBridgeDpid, gatewayIp,sourceIpAddress,sourceMacAddress,periodicRefresh));
212 final Node externalNetworkBridge = getExternalBridge(externalNetworkBridgeDpid);
213 if(externalNetworkBridge == null){
214 LOG.error("MAC address for gateway {} can not be resolved, because external bridge {} "
215 + "is not connected to controller.",gatewayIp.getValue(),externalNetworkBridgeDpid );
219 sendGatewayArpRequest(externalNetworkBridge,gatewayIp,sourceIpAddress, sourceMacAddress);
221 //Wait for MacAddress population in cache
222 return waitForMacAddress(gatewayIp);
225 private Node getExternalBridge(final Long externalNetworkBridgeDpid){
226 final String nodeName = OPENFLOW + externalNetworkBridgeDpid;
228 return getOpenFlowNode(nodeName);
231 private void sendGatewayArpRequest(final Node externalNetworkBridge,final Ipv4Address gatewayIp,
232 final Ipv4Address sourceIpAddress, final MacAddress sourceMacAddress){
233 final ArpMessageAddress senderAddress = new ArpMessageAddress(sourceMacAddress,sourceIpAddress);
235 //Build arp reply router flow
236 final Flow arpReplyToControllerFlow = createArpReplyToControllerFlow(senderAddress, gatewayIp);
238 final InstanceIdentifier<Node> nodeIid = InstanceIdentifier.builder(Nodes.class)
239 .child(Node.class, externalNetworkBridge.getKey())
241 final InstanceIdentifier<Flow> flowIid = createFlowIid(arpReplyToControllerFlow, nodeIid);
242 final NodeRef nodeRef = new NodeRef(nodeIid);
245 Future<RpcResult<AddFlowOutput>> addFlowResult = flowService.addFlow(new AddFlowInputBuilder(
246 arpReplyToControllerFlow).setFlowRef(new FlowRef(flowIid)).setNode(nodeRef).build());
247 //wait for flow installation
248 Futures.addCallback(JdkFutureAdapters.listenInPoolThread(addFlowResult),
249 new FutureCallback<RpcResult<AddFlowOutput>>() {
252 public void onSuccess(RpcResult<AddFlowOutput> result) {
253 if (!result.isSuccessful()) {
254 LOG.warn("Flow to route ARP Reply to Controller is not installed successfully : {} \nErrors: {}", flowIid,result.getErrors());
257 LOG.debug("Flow to route ARP Reply to Controller installed successfully : {}", flowIid);
259 ArpResolverMetadata gatewayArpMetadata = gatewayToArpMetadataMap.get(gatewayIp);
260 if (gatewayArpMetadata == null) {
261 LOG.warn("No metadata found for gatewayIp: {}", gatewayIp);
266 gatewayArpMetadata.setFlowToRemove(new RemoveFlowInputBuilder(arpReplyToControllerFlow).setNode(nodeRef).build());
268 //get MAC DA for ARP packets
269 MacAddress arpRequestDestMacAddress = gatewayArpMetadata.getArpRequestDestMacAddress();
271 //Send ARP request packets
272 for (NodeConnector egressNc : externalNetworkBridge.getNodeConnector()) {
273 KeyedInstanceIdentifier<NodeConnector, NodeConnectorKey> egressNcIid = nodeIid.child(
274 NodeConnector.class, new NodeConnectorKey(egressNc.getId()));
275 ListenableFuture<RpcResult<Void>> futureSendArpResult = arpSender.sendArp(
276 senderAddress, gatewayIp, arpRequestDestMacAddress, egressNcIid);
277 Futures.addCallback(futureSendArpResult, logResult(gatewayIp, egressNcIid));
282 public void onFailure(Throwable t) {
283 LOG.warn("ARP Reply to Controller flow was not created: {}", flowIid, t);
289 private ListenableFuture<MacAddress> waitForMacAddress(final Ipv4Address gatewayIp){
291 return arpWatcherWall.submit(new Callable<MacAddress>(){
294 public MacAddress call() throws Exception {
295 for(int cycle = 0;cycle < WAIT_CYCLES;cycle++){
296 //Sleep before checking mac address, so meanwhile ARP request packets
297 // will be broadcasted on the bridge.
298 Thread.sleep(PER_CYCLE_WAIT_DURATION);
299 ArpResolverMetadata arpResolverMetadata = gatewayToArpMetadataMap.get(gatewayIp);
300 if(arpResolverMetadata != null && arpResolverMetadata.getGatewayMacAddress() != null){
301 if(!arpResolverMetadata.isPeriodicRefresh()){
302 return gatewayToArpMetadataMap.remove(gatewayIp).getGatewayMacAddress();
304 return arpResolverMetadata.getGatewayMacAddress();
312 private Flow createArpReplyToControllerFlow(final ArpMessageAddress senderAddress, final Ipv4Address ipForRequestedMac) {
313 checkNotNull(senderAddress);
314 checkNotNull(ipForRequestedMac);
315 FlowBuilder arpFlow = new FlowBuilder().setTableId(TABLE_FOR_ARP_FLOW)
316 .setFlowName(ARP_REPLY_TO_CONTROLLER_FLOW_NAME)
317 .setPriority(ARP_REPLY_TO_CONTROLLER_FLOW_PRIORITY)
318 .setBufferId(OFConstants.OFP_NO_BUFFER)
321 .setCookie(new FlowCookie(BigInteger.valueOf(flowCookie.incrementAndGet())))
322 .setFlags(new FlowModFlags(false, false, false, false, false));
324 EthernetMatch ethernetMatch = ArpFlowFactory.createEthernetMatch(senderAddress.getHardwareAddress());
325 ArpMatch arpMatch = ArpFlowFactory.createArpMatch(senderAddress, ipForRequestedMac);
326 Match match = new MatchBuilder().setEthernetMatch(ethernetMatch).setLayer3Match(arpMatch).build();
327 arpFlow.setMatch(match);
328 arpFlow.setInstructions(new InstructionsBuilder().setInstruction(
329 ImmutableList.of(SEND_TO_CONTROLLER_INSTRUCTION)).build());
330 arpFlow.setId(createFlowId(ipForRequestedMac));
331 return arpFlow.build();
334 private FlowId createFlowId(Ipv4Address ipForRequestedMac) {
335 String flowId = ARP_REPLY_TO_CONTROLLER_FLOW_NAME + "|" + ipForRequestedMac.getValue();
336 return new FlowId(flowId);
339 private static InstanceIdentifier<Flow> createFlowIid(Flow flow, InstanceIdentifier<Node> nodeIid) {
340 return nodeIid.builder()
341 .augmentation(FlowCapableNode.class)
342 .child(Table.class, new TableKey(flow.getTableId()))
343 .child(Flow.class, new FlowKey(flow.getId()))
347 private FutureCallback<RpcResult<Void>> logResult(final Ipv4Address tpa,
348 final KeyedInstanceIdentifier<NodeConnector, NodeConnectorKey> egressNcIid) {
349 return new FutureCallback<RpcResult<Void>>() {
352 public void onSuccess(RpcResult<Void> result) {
353 LOG.debug("ARP Request for IP {} was sent from {}.", tpa.getValue(), egressNcIid);
357 public void onFailure(Throwable t) {
358 LOG.warn("ARP Request for IP {} was NOT sent from {}.", tpa.getValue(), egressNcIid);
364 public void onPacketReceived(PacketReceived potentialArp) {
365 Arp arp = ArpResolverUtils.getArpFrom(potentialArp);
367 if (arp.getOperation() != ArpOperation.REPLY.intValue()) {
368 LOG.trace("Packet is not ARP REPLY packet.");
371 if (LOG.isTraceEnabled()) {
372 LOG.trace("ARP REPLY received - {}", ArpUtils.getArpToStringFormat(arp));
374 NodeKey nodeKey = potentialArp.getIngress().getValue().firstKeyOf(Node.class, NodeKey.class);
375 if (nodeKey == null) {
376 LOG.info("Unknown source node of ARP packet: {}", potentialArp);
379 Ipv4Address gatewayIpAddress = ArpUtils.bytesToIp(arp.getSenderProtocolAddress());
380 MacAddress gatewayMacAddress = ArpUtils.bytesToMac(arp.getSenderHardwareAddress());
381 ArpResolverMetadata candidateGatewayIp = gatewayToArpMetadataMap.get(gatewayIpAddress);
382 if(candidateGatewayIp != null){
383 LOG.debug("Resolved MAC for Gateway Ip {} is {}",gatewayIpAddress.getValue(),gatewayMacAddress.getValue());
384 candidateGatewayIp.setGatewayMacAddress(gatewayMacAddress);
385 flowService.removeFlow(candidateGatewayIp.getFlowToRemove());
391 public void setDependencies(BundleContext bundleContext,
392 ServiceReference serviceReference) {
393 super.setDependencies(bundleContext.getServiceReference(GatewayMacResolver.class.getName()), this);
398 public void setDependencies(Object impl) {}
401 public void stopPeriodicRefresh(Ipv4Address gatewayIp) {
403 gatewayToArpMetadataMap.remove(gatewayIp);