2 * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
11 import org.apache.commons.lang3.tuple.ImmutablePair;
12 import org.apache.commons.lang3.tuple.Pair;
13 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
14 import org.opendaylight.neutron.spi.INeutronPortCRUD;
15 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
16 import org.opendaylight.neutron.spi.NeutronFloatingIP;
17 import org.opendaylight.neutron.spi.NeutronNetwork;
18 import org.opendaylight.neutron.spi.NeutronPort;
19 import org.opendaylight.neutron.spi.NeutronRouter;
20 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
21 import org.opendaylight.neutron.spi.NeutronSubnet;
22 import org.opendaylight.neutron.spi.Neutron_IPs;
23 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
24 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
25 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
26 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
29 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
30 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
32 import com.google.common.base.Preconditions;
33 import com.google.common.util.concurrent.FutureCallback;
34 import com.google.common.util.concurrent.Futures;
35 import com.google.common.util.concurrent.ListenableFuture;
37 import org.osgi.framework.BundleContext;
38 import org.osgi.framework.ServiceReference;
39 import org.slf4j.Logger;
40 import org.slf4j.LoggerFactory;
42 import java.net.InetAddress;
43 import java.net.UnknownHostException;
44 import java.util.HashMap;
45 import java.util.HashSet;
46 import java.util.ArrayList;
47 import java.util.List;
50 import java.util.concurrent.ExecutorService;
51 import java.util.concurrent.Executors;
54 * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
55 * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
56 * as well as the multi-tenant router forwarding provider.
58 public class NeutronL3Adapter implements ConfigInterface {
59 private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
61 // The implementation for each of these services is resolved by the OSGi Service Manager
62 private volatile ConfigurationService configurationService;
63 private volatile TenantNetworkManager tenantNetworkManager;
64 private volatile NodeCacheManager nodeCacheManager;
65 private volatile INeutronNetworkCRUD neutronNetworkCache;
66 private volatile INeutronSubnetCRUD neutronSubnetCache;
67 private volatile INeutronPortCRUD neutronPortCache;
68 private volatile L3ForwardingProvider l3ForwardingProvider;
69 private volatile InboundNatProvider inboundNatProvider;
70 private volatile OutboundNatProvider outboundNatProvider;
71 private volatile ArpProvider arpProvider;
72 private volatile RoutingProvider routingProvider;
73 private volatile GatewayMacResolver gatewayMacResolver;
75 private class FloatIpData {
76 // br-int of node where floating ip is associated with tenant port
77 private final Long dpid;
78 // patch port in br-int used to reach br-ex
79 private final Long ofPort;
80 // segmentation id of the net where fixed ip is instantiated
81 private final String segId;
82 // mac address assigned to neutron port of floating ip
83 private final String macAddress;
84 private final String floatingIpAddress;
85 // ip address given to tenant vm
86 private final String fixedIpAddress;
87 private final String neutronRouterMac;
89 FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
90 final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
94 this.macAddress = macAddress;
95 this.floatingIpAddress = floatingIpAddress;
96 this.fixedIpAddress = fixedIpAddress;
97 this.neutronRouterMac = neutronRouterMac;
101 private Set<String> inboundIpRewriteCache;
102 private Set<String> outboundIpRewriteCache;
103 private Set<String> outboundIpRewriteExclusionCache;
104 private Set<String> routerInterfacesCache;
105 private Set<String> staticArpEntryCache;
106 private Set<String> l3ForwardingCache;
107 private Map<String, String> networkIdToRouterMacCache;
108 private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
109 private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
110 private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
111 private Map<String, FloatIpData> floatIpDataMapCache;
112 private String externalRouterMac;
113 private Boolean enabled = false;
114 private Boolean flgDistributedARPEnabled = true;
115 private Southbound southbound;
116 private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
118 private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
119 private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
120 private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
121 private static final String OWNER_FLOATING_IP = "network:floatingip";
122 private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
124 public NeutronL3Adapter() {
125 LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
128 private void initL3AdapterMembers() {
129 Preconditions.checkNotNull(configurationService);
131 if (configurationService.isL3ForwardingEnabled()) {
132 this.inboundIpRewriteCache = new HashSet<>();
133 this.outboundIpRewriteCache = new HashSet<>();
134 this.outboundIpRewriteExclusionCache = new HashSet<>();
135 this.routerInterfacesCache = new HashSet<>();
136 this.staticArpEntryCache = new HashSet<>();
137 this.l3ForwardingCache = new HashSet<>();
138 this.networkIdToRouterMacCache = new HashMap<>();
139 this.networkIdToRouterIpListCache = new HashMap<>();
140 this.subnetIdToRouterInterfaceCache = new HashMap<>();
141 this.neutronPortToDpIdCache = new HashMap<>();
142 this.floatIpDataMapCache = new HashMap<>();
144 this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
145 if (this.externalRouterMac == null) {
146 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
150 LOG.info("OVSDB L3 forwarding is enabled");
151 if (configurationService.isDistributedArpDisabled()) {
152 this.flgDistributedARPEnabled = false;
153 LOG.info("Distributed ARP responder is disabled");
155 LOG.debug("Distributed ARP responder is enabled");
158 LOG.debug("OVSDB L3 forwarding is disabled");
163 // Callbacks from OVSDB's northbound handlers
167 * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
168 * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
169 * subscribe so that the mac address associated to the gateway ip address is available.
171 * @param externalRouterMacUpdate The mac address to be associated to the gateway.
173 public void updateExternalRouterMac(final String externalRouterMacUpdate) {
174 Preconditions.checkNotNull(externalRouterMacUpdate);
176 flushExistingIpRewrite();
177 this.externalRouterMac = externalRouterMacUpdate;
178 rebuildExistingIpRewrite();
184 * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
185 * @param subnet An instance of NeutronSubnet object.
187 public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
188 LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
192 * Process the port event as a router interface event.
193 * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
194 * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
196 * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
197 * @param neutronPort An instance of NeutronPort object.
199 public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
200 LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
205 final boolean isDelete = action == Action.DELETE;
207 if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
209 Node externalBridgeNode = getExternalBridgeNode();
210 if(externalBridgeNode != null){
211 LOG.info("Port {} is network router gateway interface, "
212 + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
213 this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
215 LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
218 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
220 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
221 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
223 if (externalSubnet != null &&
224 externalSubnet.getIpVersion() == 4) {
225 gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
231 // Treat the port event as a router interface event if the port belongs to router. This is a
232 // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
234 if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
235 neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
237 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
238 NeutronRouter_Interface neutronRouterInterface =
239 new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
240 // id of router interface to be same as subnet
241 neutronRouterInterface.setID(neutronIP.getSubnetUUID());
242 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
244 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
247 // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
248 // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
249 // need to do this check here because a router interface is not added to a node until tenant becomes needed
252 if (!isDelete && neutronPort.getFixedIPs() != null) {
253 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
254 NeutronRouter_Interface neutronRouterInterface =
255 subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
256 if (neutronRouterInterface != null) {
257 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
261 this.updateL3ForNeutronPort(neutronPort, isDelete);
268 * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
269 * @param neutronRouter An instance of NeutronRouter object.
271 public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
272 LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
276 * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
277 * delete the ports on the same subnet.
279 * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
280 * @param neutronRouter An instance of NeutronRouter object.
281 * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
283 public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
284 final NeutronRouter_Interface neutronRouterInterface,
286 LOG.debug("Router interface {} got event {}. Subnet {}",
287 neutronRouterInterface.getPortUUID(),
289 neutronRouterInterface.getSubnetUUID());
294 final boolean isDelete = action == Action.DELETE;
296 this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
298 // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
299 // see if they are affected by l3
301 for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
302 boolean currPortShouldBeDeleted = false;
303 // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
305 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
306 if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
307 currPortShouldBeDeleted = true;
312 this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
317 * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
318 * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
319 * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
320 * way around) on OpenFlow Table 100.
322 * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
323 * @param neutronFloatingIP An {@link org.opendaylight.neutron.spi.NeutronFloatingIP} instance of NeutronFloatingIP object.
325 public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
327 Preconditions.checkNotNull(neutronFloatingIP);
329 LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
330 neutronFloatingIP.getFixedIPAddress(),
331 neutronFloatingIP.getFloatingIPAddress(),
332 neutronFloatingIP.getFloatingNetworkUUID());
339 // Consider action to be delete if getFixedIPAddress is null
341 if (neutronFloatingIP.getFixedIPAddress() == null) {
342 action = Action.DELETE;
347 // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
349 if (action != Action.DELETE) {
350 // must be first, as it updates floatIpDataMapCache
351 programFlowsForFloatingIPArpAdd(neutronFloatingIP);
353 programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
354 programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
356 programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
357 programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
359 // must be last, as it updates floatIpDataMapCache
360 programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
365 * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
366 * ip, otherwise for newer one.
368 private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
369 Preconditions.checkNotNull(neutronFloatingIP);
371 final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
373 LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
374 action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
377 programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
382 * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
383 * ip, otherwise for newer one.
385 private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
386 Preconditions.checkNotNull(neutronFloatingIP);
388 final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
390 LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
391 action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
394 programOutboundIpRewriteStage1(fid, action);
397 private void flushExistingIpRewrite() {
398 for (FloatIpData fid : floatIpDataMapCache.values()) {
399 programOutboundIpRewriteStage1(fid, Action.DELETE);
403 private void rebuildExistingIpRewrite() {
404 for (FloatIpData fid : floatIpDataMapCache.values()) {
405 programOutboundIpRewriteStage1(fid, Action.ADD);
410 * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
411 * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
414 private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
415 Preconditions.checkNotNull(neutronFloatingIP);
416 Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
417 Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
419 if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
420 LOG.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
421 neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
425 // find bridge Node where floating ip is configured by looking up cache for its port
426 final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
427 final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
428 final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
429 final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
430 final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
431 final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
433 final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
434 final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
435 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
436 final String providerSegmentationId = tenantNeutronNetwork != null ?
437 tenantNeutronNetwork.getProviderSegmentationID() : null;
438 final String neutronRouterMac = tenantNeutronNetwork != null ?
439 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
441 if (nodeIfPair == null || neutronTenantPortUuid == null ||
442 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
443 floatingIpMac == null || floatingIpMac.isEmpty() ||
444 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
445 LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
448 neutronPortForFloatIp,
449 neutronTenantPortUuid,
450 providerSegmentationId,
456 // get ofport for patch port in br-int
457 final Long dpId = nodeIfPair.getLeft();
458 final Long ofPort = findOFPortForExtPatch(dpId);
459 if (ofPort == null) {
460 LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
465 // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
467 if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
469 final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
470 floatingIpAddress, fixedIpAddress, neutronRouterMac);
471 floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
472 LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
473 neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
474 floatingIpMac, ofPort, providerSegmentationId, dpId);
478 private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
479 final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
480 if (floatIpData == null) {
481 LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
485 if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
486 floatIpData.floatingIpAddress, Action.DELETE)) {
487 floatIpDataMapCache.remove(neutronFloatingIPUuid);
488 LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
489 floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
493 private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
494 for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
495 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
496 neutronPort.getDeviceID().equals(floatingIpUuid)) {
503 private Long findOFPortForExtPatch(Long dpId) {
504 final String brInt = configurationService.getIntegrationBridgeName();
505 final String brExt = configurationService.getExternalBridgeName();
506 final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
508 Preconditions.checkNotNull(dpId);
509 Preconditions.checkNotNull(portNameInt);
511 final long dpidPrimitive = dpId;
512 for (Node node : nodeCacheManager.getBridgeNodes()) {
513 if (dpidPrimitive == southbound.getDataPathId(node)) {
514 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
515 southbound.getTerminationPointOfBridge(node, portNameInt);
516 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
525 * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
526 * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronFloatingIP object.
528 public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
529 LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
533 // Callbacks from OVSDB's southbound handler
538 * @param bridgeNode An instance of Node object.
539 * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
540 * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
541 * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronNetwork
543 * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
545 public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
546 final NeutronNetwork neutronNetwork, Action action) {
547 LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
548 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
553 final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
554 final Long dpId = getDpidForIntegrationBridge(bridgeNode);
555 final Uuid interfaceUuid = intf.getInterfaceUuid();
557 LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
558 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
560 if (neutronPort != null) {
561 final String neutronPortUuid = neutronPort.getPortUUID();
563 if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
564 dpId != null && interfaceUuid != null) {
565 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
568 handleNeutronPortEvent(neutronPort, action);
571 if (action == Action.DELETE && interfaceUuid != null) {
572 handleInterfaceEventDelete(intf, dpId);
576 private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
577 neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
578 LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
579 neutronPortUuid, dpId, interfaceUuid.getValue());
582 private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
583 // Remove entry from neutronPortToDpIdCache based on interface uuid
584 for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
585 final String currPortUuid = entry.getKey();
586 if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
587 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
588 currPortUuid, dpId, intf.getInterfaceUuid().getValue());
589 neutronPortToDpIdCache.remove(currPortUuid);
598 private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
600 final String networkUUID = neutronPort.getNetworkUUID();
601 final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
603 // If there is no router interface handling the networkUUID, we are done
604 if (routerMacAddress == null || routerMacAddress.isEmpty()) {
608 // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
609 // router interface are handled via handleNeutronRouterInterfaceEvent.
610 if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
614 final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
615 final String providerSegmentationId = neutronNetwork != null ?
616 neutronNetwork.getProviderSegmentationID() : null;
617 final String tenantMac = neutronPort.getMacAddress();
619 if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
620 tenantMac == null || tenantMac.isEmpty()) {
621 // done: go no further w/out all the info needed...
625 final Action action = isDelete ? Action.DELETE : Action.ADD;
626 List<Node> nodes = nodeCacheManager.getBridgeNodes();
627 if (nodes.isEmpty()) {
628 LOG.trace("updateL3ForNeutronPort has no nodes to work with");
630 for (Node node : nodes) {
631 final Long dpid = getDpidForIntegrationBridge(node);
636 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
637 final String tenantIpStr = neutronIP.getIpAddress();
638 if (tenantIpStr.isEmpty()) {
642 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
643 // still needed when routing to subnets non-local to node (bug 2076).
644 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
646 // Configure distributed ARP responder
647 if (flgDistributedARPEnabled) {
648 programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
654 private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
655 String macAddress, String ipStr,
656 Action actionForNode) {
657 // Based on the local cache, figure out whether programming needs to occur. To do this, we
658 // will look at desired action for node.
660 final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
661 final boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
663 if (actionForNode == Action.DELETE && !isProgrammed) {
664 LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
665 node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
668 if (actionForNode == Action.ADD && isProgrammed) {
669 LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
670 node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
674 Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
675 macAddress, ipStr, actionForNode);
676 if (status.isSuccess()) {
678 if (actionForNode == Action.ADD) {
679 l3ForwardingCache.add(cacheKey);
681 l3ForwardingCache.remove(cacheKey);
686 private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
689 Action actionForNode) {
692 InetAddress inetAddress = InetAddress.getByName(address);
693 status = l3ForwardingProvider == null ?
694 new Status(StatusCode.SUCCESS) :
695 l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
696 inetAddress, macAddress, actionForNode);
697 } catch (UnknownHostException e) {
698 status = new Status(StatusCode.BADREQUEST);
701 if (status.isSuccess()) {
702 LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
703 l3ForwardingProvider == null ? "skipped" : "programmed",
704 macAddress, address, node.getNodeId().getValue(), actionForNode);
706 LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
707 macAddress, address, node.getNodeId().getValue(), actionForNode, status);
714 private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
716 Preconditions.checkNotNull(destNeutronRouterInterface);
718 final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
719 String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
720 List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
721 final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
722 final NeutronNetwork neutronNetwork = subnet != null ?
723 neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
724 final String destinationSegmentationId = neutronNetwork != null ?
725 neutronNetwork.getProviderSegmentationID() : null;
726 final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
727 final String cidr = subnet != null ? subnet.getCidr() : null;
728 final int mask = getMaskLenFromCidr(cidr);
730 LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
731 destNeutronRouterInterface, isDelete);
733 // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
735 if (neutronNetwork != null) {
736 if (macAddress == null || macAddress.isEmpty()) {
737 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
739 if (ipList == null || ipList.isEmpty()) {
740 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
744 if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
745 cidr == null || cidr.isEmpty() ||
746 macAddress == null || macAddress.isEmpty() ||
747 ipList == null || ipList.isEmpty()) {
748 LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{} ip:{}",
749 destinationSegmentationId, cidr, macAddress, ipList);
750 // done: go no further w/out all the info needed...
754 final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
756 // Keep cache for finding router's mac from network uuid -- add
759 networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
760 networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
761 subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
764 List<Node> nodes = nodeCacheManager.getBridgeNodes();
765 if (nodes.isEmpty()) {
766 LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
768 for (Node node : nodes) {
769 final Long dpid = getDpidForIntegrationBridge(node);
774 for (Neutron_IPs neutronIP : ipList) {
775 final String ipStr = neutronIP.getIpAddress();
776 if (ipStr.isEmpty()) {
777 LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
778 node.getNodeId().getValue(), ipStr);
782 // Iterate through all other interfaces and add/remove reflexive flows to this interface
784 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
785 programFlowsForNeutronRouterInterfacePair(node, dpid,
786 srcNeutronRouterInterface, destNeutronRouterInterface,
787 neutronNetwork, destinationSegmentationId,
788 macAddress, ipStr, mask, actionForNode,
789 true /*isReflexsive*/);
793 programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
796 // Enable ARP responder by default, because router interface needs to be responded always.
797 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
800 // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
801 // for the external neutron networks.
804 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
805 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
809 // Keep cache for finding router's mac from network uuid -- remove
812 networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
813 networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
814 subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
818 private void programFlowForNetworkFromExternal(final Node node,
820 final String destinationSegmentationId,
821 final String dstMacAddress,
822 final String destIpStr,
824 final Action actionForNode) {
825 programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
826 dstMacAddress, destIpStr, destMask, actionForNode);
829 private void programFlowsForNeutronRouterInterfacePair(final Node node,
831 final NeutronRouter_Interface srcNeutronRouterInterface,
832 final NeutronRouter_Interface dstNeutronRouterInterface,
833 final NeutronNetwork dstNeutronNetwork,
834 final String destinationSegmentationId,
835 final String dstMacAddress,
836 final String destIpStr,
838 final Action actionForNode,
839 Boolean isReflexsive) {
840 Preconditions.checkNotNull(srcNeutronRouterInterface);
841 Preconditions.checkNotNull(dstNeutronRouterInterface);
843 final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
844 if (sourceSubnetId == null) {
845 LOG.error("Could not get provider Subnet ID from router interface {}",
846 srcNeutronRouterInterface.getID());
850 final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
851 final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
852 if (sourceNetworkId == null) {
853 LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
857 final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
858 if (sourceNetwork == null) {
859 LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
863 if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
864 // Isolate subnets from different tenants within the same router
867 final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
868 if (sourceSegmentationId == null) {
869 LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
872 if (sourceSegmentationId.equals(destinationSegmentationId)) {
877 programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
878 dstMacAddress, destIpStr, destMask, actionForNode);
880 // Flip roles src->dst; dst->src
882 final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
883 final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
884 final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
885 final String cidr2 = sourceSubnet.getCidr();
886 final int mask2 = getMaskLenFromCidr(cidr2);
888 if (cidr2 == null || cidr2.isEmpty() ||
889 macAddress2 == null || macAddress2.isEmpty() ||
890 ipList2 == null || ipList2.isEmpty()) {
891 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
892 sourceSegmentationId, cidr2, macAddress2, ipList2);
893 // done: go no further w/out all the info needed...
897 for (Neutron_IPs neutronIP2 : ipList2) {
898 final String ipStr2 = neutronIP2.getIpAddress();
899 if (ipStr2.isEmpty()) {
902 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
903 srcNeutronRouterInterface,
904 sourceNetwork, sourceSegmentationId,
905 macAddress2, ipStr2, mask2, actionForNode,
906 false /*isReflexsive*/);
911 private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
912 String destinationSegmentationId,
913 String macAddress, String ipStr, int mask,
914 Action actionForNode) {
915 // Based on the local cache, figure out whether programming needs to occur. To do this, we
916 // will look at desired action for node.
918 final String cacheKey = node.getNodeId().getValue() + ":" +
919 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
920 ipStr + "/" + Integer.toString(mask);
921 final boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
923 if (actionForNode == Action.DELETE && !isProgrammed) {
924 LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
925 " action {} is already done",
926 node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
927 macAddress, ipStr, mask, actionForNode);
930 if (actionForNode == Action.ADD && isProgrammed) {
931 LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
932 " action {} is already done",
933 node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
934 macAddress, ipStr, mask, actionForNode);
938 Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
939 macAddress, ipStr, mask, actionForNode);
940 if (status.isSuccess()) {
942 if (actionForNode == Action.ADD) {
943 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
944 routerInterfacesCache.add(cacheKey);
946 // TODO: multiTenantAwareRouter.removeInterface(...);
947 routerInterfacesCache.remove(cacheKey);
952 private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
953 String destinationSegmentationId,
955 String address, int mask,
956 Action actionForNode) {
959 InetAddress inetAddress = InetAddress.getByName(address);
960 status = routingProvider == null ?
961 new Status(StatusCode.SUCCESS) :
962 routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
963 macAddress, inetAddress, mask, actionForNode);
964 } catch (UnknownHostException e) {
965 status = new Status(StatusCode.BADREQUEST);
968 if (status.isSuccess()) {
969 LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
970 routingProvider == null ? "skipped" : "programmed",
971 macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
974 LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
975 macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
976 actionForNode, status);
981 private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
982 String macAddress, String ipStr,
984 // Based on the local cache, figure out whether programming needs to occur. To do this, we
985 // will look at desired action for node.
987 final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
988 final boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
990 if (action == Action.DELETE && !isProgrammed) {
991 LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
992 dpid, segOrOfPort, macAddress, ipStr, action);
995 if (action == Action.ADD && isProgrammed) {
996 LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
997 dpid, segOrOfPort, macAddress, ipStr, action);
1001 Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1002 if (status.isSuccess()) {
1004 if (action == Action.ADD) {
1005 staticArpEntryCache.add(cacheKey);
1007 staticArpEntryCache.remove(cacheKey);
1014 private Status programStaticArpStage2(Long dpid,
1021 InetAddress inetAddress = InetAddress.getByName(address);
1022 status = arpProvider == null ?
1023 new Status(StatusCode.SUCCESS) :
1024 arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1025 macAddress, inetAddress, action);
1026 } catch (UnknownHostException e) {
1027 status = new Status(StatusCode.BADREQUEST);
1030 if (status.isSuccess()) {
1031 LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1032 arpProvider == null ? "skipped" : "programmed",
1033 macAddress, address, dpid, segOrOfPort, action);
1035 LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1036 macAddress, address, dpid, segOrOfPort, action, status);
1041 private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1042 String matchAddress, String rewriteAddress,
1044 // Based on the local cache, figure out whether programming needs to occur. To do this, we
1045 // will look at desired action for node.
1047 final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
1048 final boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
1050 if (action == Action.DELETE && !isProgrammed) {
1051 LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1052 " action {} is already done",
1053 dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1056 if (action == Action.ADD && isProgrammed) {
1057 LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1058 " action is already done",
1059 dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1063 Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1064 rewriteAddress, action);
1065 if (status.isSuccess()) {
1067 if (action == Action.ADD) {
1068 inboundIpRewriteCache.add(cacheKey);
1070 inboundIpRewriteCache.remove(cacheKey);
1077 private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1078 String matchAddress, String rewriteAddress,
1082 InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1083 InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1084 status = inboundNatProvider == null ?
1085 new Status(StatusCode.SUCCESS) :
1086 inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1087 inetMatchAddress, inetRewriteAddress,
1089 } catch (UnknownHostException e) {
1090 status = new Status(StatusCode.BADREQUEST);
1093 if (status.isSuccess()) {
1094 final boolean isSkipped = inboundNatProvider == null;
1095 LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1096 isSkipped ? "skipped" : "programmed",
1097 dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1099 LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1101 dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1107 private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1108 Action actionForRewriteExclusion) {
1109 // Based on the local cache, figure out whether programming needs to occur. To do this, we
1110 // will look at desired action for node.
1112 final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1113 final boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1115 if (actionForRewriteExclusion == Action.DELETE && !isProgrammed) {
1116 LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1117 node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1120 if (actionForRewriteExclusion == Action.ADD && isProgrammed) {
1121 LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1122 node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1126 Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1127 actionForRewriteExclusion);
1128 if (status.isSuccess()) {
1130 if (actionForRewriteExclusion == Action.ADD) {
1131 outboundIpRewriteExclusionCache.add(cacheKey);
1133 outboundIpRewriteExclusionCache.remove(cacheKey);
1138 private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1139 Action actionForNode) {
1140 final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1141 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1143 if (status.isSuccess()) {
1144 final boolean isSkipped = outboundNatProvider == null;
1145 LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1146 isSkipped ? "skipped" : "programmed",
1147 cidr, node.getNodeId().getValue(), actionForNode);
1149 LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1150 cidr, node.getNodeId().getValue(), actionForNode, status);
1155 private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1156 // Based on the local cache, figure out whether programming needs to occur. To do this, we
1157 // will look at desired action for node.
1159 final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1160 final boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1162 if (action == Action.DELETE && !isProgrammed) {
1163 LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1165 fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1168 if (action == Action.ADD && isProgrammed) {
1169 LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1171 fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1175 Status status = this.programOutboundIpRewriteStage2(fid, action);
1176 if (status.isSuccess()) {
1178 if (action == Action.ADD) {
1179 outboundIpRewriteCache.add(cacheKey);
1181 outboundIpRewriteCache.remove(cacheKey);
1186 private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1189 InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1190 InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1191 status = outboundNatProvider == null ?
1192 new Status(StatusCode.SUCCESS) :
1193 outboundNatProvider.programIpRewriteRule(
1194 fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1195 this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1196 } catch (UnknownHostException e) {
1197 status = new Status(StatusCode.BADREQUEST);
1200 if (status.isSuccess()) {
1201 final boolean isSkipped = outboundNatProvider == null;
1202 LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1204 isSkipped ? "skipped" : "programmed",
1205 fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1207 LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1208 " action {} status:{}",
1209 fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1214 private int getMaskLenFromCidr(String cidr) {
1218 String[] splits = cidr.split("/");
1219 if (splits.length != 2) {
1225 result = Integer.parseInt(splits[1].trim());
1226 } catch (NumberFormatException nfe) {
1232 private Long getDpidForIntegrationBridge(Node node) {
1233 // Check if node is integration bridge; and only then return its dpid
1234 if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1235 return southbound.getDataPathId(node);
1240 private Long getDpidForExternalBridge(Node node) {
1241 // Check if node is integration bridge; and only then return its dpid
1242 if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1243 return southbound.getDataPathId(node);
1248 private Node getExternalBridgeNode(){
1249 //Pickup the first node that has external bridge (br-ex).
1250 //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1251 //the external network is reachable from every br-ex
1252 // TODO: Consider other deployment scenario, and thing of better solution.
1253 List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1254 for(Node node : allBridges){
1255 if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1262 private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1263 for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1264 String subnetUUID = neutronIPs.getSubnetUUID();
1265 NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1266 if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1269 if (extSubnet == null) {
1270 // TODO: when subnet is created, try again.
1271 LOG.debug("subnet {} in not found", subnetUUID);
1277 public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1279 Preconditions.checkNotNull(node);
1280 Preconditions.checkNotNull(gatewayPort);
1281 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1283 if(externalNetwork != null){
1284 if(externalNetwork.isRouterExternal()){
1285 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1287 // TODO: address IPv6 case.
1288 if (externalSubnet != null &&
1289 externalSubnet.getIpVersion() == 4) {
1290 LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
1291 ListenableFuture<MacAddress> gatewayMacAddress =
1292 gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1293 new Ipv4Address(externalSubnet.getGatewayIP()),
1294 new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1295 new MacAddress(gatewayPort.getMacAddress()),
1297 if(gatewayMacAddress != null){
1298 Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1300 public void onSuccess(MacAddress result) {
1302 if(!result.getValue().equals(externalRouterMac)){
1303 updateExternalRouterMac(result.getValue());
1304 LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
1307 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1312 public void onFailure(Throwable t) {
1313 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1315 }, gatewayMacResolverPool);
1318 LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1322 LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1327 * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1329 * @param ofPort the OF port number
1330 * @return the string with encoded OF port (example format "OFPort|999")
1332 public static String encodeExcplicitOFPort(Long ofPort) {
1333 return "OFPort|" + ofPort.toString();
1337 public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1338 tenantNetworkManager =
1339 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1340 configurationService =
1341 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1343 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1344 inboundNatProvider =
1345 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1346 outboundNatProvider =
1347 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1349 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1350 l3ForwardingProvider =
1351 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1353 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1355 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1356 gatewayMacResolver =
1357 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1358 initL3AdapterMembers();
1362 public void setDependencies(Object impl) {
1363 if (impl instanceof INeutronNetworkCRUD) {
1364 neutronNetworkCache = (INeutronNetworkCRUD)impl;
1365 } else if (impl instanceof INeutronPortCRUD) {
1366 neutronPortCache = (INeutronPortCRUD)impl;
1367 } else if (impl instanceof INeutronSubnetCRUD) {
1368 neutronSubnetCache = (INeutronSubnetCRUD)impl;
1369 } else if (impl instanceof ArpProvider) {
1370 arpProvider = (ArpProvider)impl;
1371 } else if (impl instanceof InboundNatProvider) {
1372 inboundNatProvider = (InboundNatProvider)impl;
1373 } else if (impl instanceof OutboundNatProvider) {
1374 outboundNatProvider = (OutboundNatProvider)impl;
1375 } else if (impl instanceof RoutingProvider) {
1376 routingProvider = (RoutingProvider)impl;
1377 } else if (impl instanceof L3ForwardingProvider) {
1378 l3ForwardingProvider = (L3ForwardingProvider)impl;
1379 }else if (impl instanceof GatewayMacResolver) {
1380 gatewayMacResolver = (GatewayMacResolver)impl;