2 * Copyright IBM Corporation, 2013. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.controller.samples.loadbalancer.internal;
10 import java.net.InetAddress;
11 import java.net.UnknownHostException;
12 import java.util.ArrayList;
13 import java.util.Dictionary;
14 import java.util.List;
17 import org.apache.felix.dm.Component;
18 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
19 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
20 import org.opendaylight.controller.hosttracker.HostIdFactory;
21 import org.opendaylight.controller.hosttracker.IHostId;
22 import org.opendaylight.controller.hosttracker.IfIptoHost;
23 import org.opendaylight.controller.hosttracker.hostAware.HostNodeConnector;
24 import org.opendaylight.controller.sal.action.Action;
25 import org.opendaylight.controller.sal.action.Output;
26 import org.opendaylight.controller.sal.action.SetDlDst;
27 import org.opendaylight.controller.sal.action.SetDlSrc;
28 import org.opendaylight.controller.sal.action.SetNwDst;
29 import org.opendaylight.controller.sal.action.SetNwSrc;
30 import org.opendaylight.controller.sal.core.Node;
31 import org.opendaylight.controller.sal.core.NodeConnector;
32 import org.opendaylight.controller.sal.core.Path;
33 import org.opendaylight.controller.sal.flowprogrammer.Flow;
34 import org.opendaylight.controller.sal.match.Match;
35 import org.opendaylight.controller.sal.match.MatchType;
36 import org.opendaylight.controller.sal.packet.Ethernet;
37 import org.opendaylight.controller.sal.packet.IDataPacketService;
38 import org.opendaylight.controller.sal.packet.IListenDataPacket;
39 import org.opendaylight.controller.sal.packet.IPv4;
40 import org.opendaylight.controller.sal.packet.Packet;
41 import org.opendaylight.controller.sal.packet.PacketResult;
42 import org.opendaylight.controller.sal.packet.RawPacket;
43 import org.opendaylight.controller.sal.routing.IRouting;
44 import org.opendaylight.controller.sal.utils.EtherTypes;
45 import org.opendaylight.controller.sal.utils.GlobalConstants;
46 import org.opendaylight.controller.sal.utils.IPProtocols;
47 import org.opendaylight.controller.samples.loadbalancer.ConfigManager;
48 import org.opendaylight.controller.samples.loadbalancer.IConfigManager;
49 import org.opendaylight.controller.samples.loadbalancer.LBConst;
50 import org.opendaylight.controller.samples.loadbalancer.LBUtil;
51 import org.opendaylight.controller.samples.loadbalancer.entities.Client;
52 import org.opendaylight.controller.samples.loadbalancer.entities.Pool;
53 import org.opendaylight.controller.samples.loadbalancer.entities.PoolMember;
54 import org.opendaylight.controller.samples.loadbalancer.entities.VIP;
55 import org.opendaylight.controller.samples.loadbalancer.policies.RandomLBPolicy;
56 import org.opendaylight.controller.samples.loadbalancer.policies.RoundRobinLBPolicy;
57 import org.slf4j.Logger;
58 import org.slf4j.LoggerFactory;
61 * This class is the main class that represents the load balancer service. This
62 * is a sample load balancer application that balances traffic to backend
63 * servers based on the source address and source port on each incoming packet.
64 * The service reactively installs OpenFlow rules to direct all packets with a
65 * specific source address and source port to one of the appropriate backend
66 * servers. The servers may be chosen using a round robin policy or a random
67 * policy. This service can be configured via a REST APIs which are similar to
68 * the OpenStack Quantum LBaaS (Load-balancer-as-a-Service) v1.0 API proposal
69 * (http://wiki.openstack.org/Quantum/LBaaS)
71 * To use this service, a virtual IP (or VIP) should be exposed to the clients
72 * of this service and used as the destination address. A VIP is a entity that
73 * comprises of a virtual IP, port and protocol (TCP or UDP). Assumptions: 1.
74 * One or more VIPs may be mapped to the same server pool. All VIPs that share
75 * the same pool must also share the same load balancing policy (random or round
78 * 2. Only one server pool can be be assigned to a VIP.
80 * 3. All flow rules are installed with an idle timeout of 5 seconds.
82 * 4. Packets to a VIP must leave the OpenFlow cluster from the same switch from
83 * where it entered it.
85 * 5. When you delete a VIP or a server pool or a server from a pool, the
86 * service does not delete the flow rules it has already installed. The flow
87 * rules should automatically time out after the idle timeout of 5 seconds.
90 public class LoadBalancerService implements IListenDataPacket, IConfigManager {
95 private static Logger lbsLogger = LoggerFactory.getLogger(LoadBalancerService.class);
98 * Single instance of the configuration manager. Application passes this
99 * reference to all the new policies implemented for load balancing.
101 private static ConfigManager configManager = new ConfigManager();
104 * Round robing policy instance. Need to implement factory patterns to get
107 private static RoundRobinLBPolicy rrLBMethod = new RoundRobinLBPolicy(configManager);
110 * Random policy instance.
112 private static RandomLBPolicy ranLBMethod = new RandomLBPolicy(configManager);
115 * Reference to the data packet service
117 private IDataPacketService dataPacketService = null;
120 * Reference to the host tracker service
122 private IfIptoHost hostTracker;
125 * Reference to the forwarding manager
127 private IForwardingRulesManager ruleManager;
130 * Reference to the routing service
132 private IRouting routing;
135 * Load balancer application installs all flows with priority 2.
137 private static short LB_IPSWITCH_PRIORITY = 2;
140 * Name of the container where this application will register.
142 private String containerName = null;
145 * Set/unset methods for the service instance that load balancer service
148 public String getContainerName() {
149 if (containerName == null) {
150 return GlobalConstants.DEFAULT.toString();
152 return containerName;
155 void setDataPacketService(IDataPacketService s) {
156 this.dataPacketService = s;
159 void unsetDataPacketService(IDataPacketService s) {
160 if (this.dataPacketService == s) {
161 this.dataPacketService = null;
165 public void setRouting(IRouting routing) {
166 this.routing = routing;
169 public void unsetRouting(IRouting routing) {
170 if (this.routing == routing) {
175 public void setHostTracker(IfIptoHost hostTracker) {
176 lbsLogger.debug("Setting HostTracker");
177 this.hostTracker = hostTracker;
180 public void unsetHostTracker(IfIptoHost hostTracker) {
181 if (this.hostTracker == hostTracker) {
182 this.hostTracker = null;
186 public void setForwardingRulesManager(IForwardingRulesManager forwardingRulesManager) {
187 lbsLogger.debug("Setting ForwardingRulesManager");
188 this.ruleManager = forwardingRulesManager;
191 public void unsetForwardingRulesManager(IForwardingRulesManager forwardingRulesManager) {
192 if (this.ruleManager == forwardingRulesManager) {
193 this.ruleManager = null;
198 * This method receives first packet of flows for which there is no matching
199 * flow rule installed on the switch. IP addresses used for VIPs are not
200 * supposed to be used by any real/virtual host in the network. Hence, any
201 * forwarding/routing service will not install any flows rules matching
202 * these VIPs. This ensures that all the flows destined for VIPs will not
203 * find a match in the switch and will be forwarded to the load balancing
204 * service. Service will decide where to route this traffic based on the
205 * load balancing policy of the VIP's attached pool and will install
206 * appropriate flow rules in a reactive manner.
209 public PacketResult receiveDataPacket(RawPacket inPkt) {
212 return PacketResult.IGNORED;
215 Packet formattedPak = this.dataPacketService.decodeDataPacket(inPkt);
217 if (formattedPak instanceof Ethernet) {
218 byte[] vipMacAddr = ((Ethernet) formattedPak).getDestinationMACAddress();
219 Object ipPkt = formattedPak.getPayload();
221 if (ipPkt instanceof IPv4) {
223 lbsLogger.debug("Packet recieved from switch : {}", inPkt.getIncomingNodeConnector().getNode()
225 IPv4 ipv4Pkt = (IPv4) ipPkt;
226 if (IPProtocols.getProtocolName(ipv4Pkt.getProtocol()).equals(IPProtocols.TCP.toString())
227 || IPProtocols.getProtocolName(ipv4Pkt.getProtocol()).equals(IPProtocols.UDP.toString())) {
229 lbsLogger.debug("Packet protocol : {}", IPProtocols.getProtocolName(ipv4Pkt.getProtocol()));
230 Client client = new LBUtil().getClientFromPacket(ipv4Pkt);
231 VIP vip = new LBUtil().getVIPFromPacket(ipv4Pkt);
233 if (configManager.vipExists(vip)) {
234 VIP vipWithPoolName = configManager.getVIPWithPoolName(vip);
235 String poolMemberIp = null;
236 if (vipWithPoolName.getPoolName() == null) {
237 lbsLogger.error("No pool attached. Please attach pool with the VIP -- {}", vip);
238 return PacketResult.IGNORED;
240 if (configManager.getPool(vipWithPoolName.getPoolName()).getLbMethod()
241 .equalsIgnoreCase(LBConst.ROUND_ROBIN_LB_METHOD)) {
243 poolMemberIp = rrLBMethod.getPoolMemberForClient(client, vipWithPoolName);
246 if (configManager.getPool(vipWithPoolName.getPoolName()).getLbMethod()
247 .equalsIgnoreCase(LBConst.RANDOM_LB_METHOD)) {
248 poolMemberIp = ranLBMethod.getPoolMemberForClient(client, vipWithPoolName);
253 Node clientNode = inPkt.getIncomingNodeConnector().getNode();
254 // HostTracker hosts db key scheme implementation
255 IHostId id = HostIdFactory.create(InetAddress.getByName(poolMemberIp), null);
256 HostNodeConnector hnConnector = this.hostTracker.hostFind(id);
258 Node destNode = hnConnector.getnodeconnectorNode();
260 lbsLogger.debug("Client is connected to switch : {}", clientNode.toString());
262 .debug("Destination pool machine is connected to switch : {}", destNode.toString());
264 // Get path between both the nodes
265 NodeConnector forwardPort = null;
267 if (clientNode.getNodeIDString().equals(destNode.getNodeIDString())) {
269 forwardPort = hnConnector.getnodeConnector();
272 .trace("Both source (client) and destination pool machine is connected to same switch nodes. Respective ports are - {},{}",
273 forwardPort, inPkt.getIncomingNodeConnector());
277 Path route = this.routing.getRoute(clientNode, destNode);
279 lbsLogger.trace("Path between source (client) and destination switch nodes : {}",
282 forwardPort = route.getEdges().get(0).getTailNodeConnector();
286 if (installLoadBalancerFlow(client, vip, clientNode, poolMemberIp,
287 hnConnector.getDataLayerAddressBytes(), forwardPort,
288 LBConst.FORWARD_DIRECTION_LB_FLOW)) {
289 lbsLogger.trace("Traffic from client : {} will be routed " + "to pool machine : {}",
290 client, poolMemberIp);
292 lbsLogger.error("Not able to route traffic from client : {}", client);
295 if (installLoadBalancerFlow(client, vip, clientNode, poolMemberIp, vipMacAddr,
296 inPkt.getIncomingNodeConnector(), LBConst.REVERSE_DIRECTION_LB_FLOW)) {
297 lbsLogger.trace("Flow rule installed to change the source ip/mac from "
298 + "pool machine ip {} to VIP {} for traffic coming pool machine", poolMemberIp,
301 lbsLogger.error("Not able to route traffic from client : {}", client);
303 } catch (UnknownHostException e) {
304 lbsLogger.error("Pool member not found in the network : {}", e.getMessage());
305 lbsLogger.error("", e);
311 return PacketResult.IGNORED;
315 * This method installs the flow rule for routing the traffic between two
318 * @param source Traffic is sent by this source
320 * @param dest Traffic is destined to this destination (VIP)
322 * @param sourceSwitch Switch from where controller received the packet
324 * @param destMachineIp IP address of the pool member where traffic needs to
327 * @param destMachineMac MAC address of the pool member where traffic needs
330 * @param outport Use this port to send out traffic
332 * @param flowDirection FORWARD_DIRECTION_LB_FLOW or
333 * REVERSE_DIRECTION_LB_FLOW
335 * @return true If flow installation was successful false else
337 * @throws UnknownHostException
339 private boolean installLoadBalancerFlow(Client source, VIP dest, Node sourceSwitch, String destMachineIp,
340 byte[] destMachineMac, NodeConnector outport, int flowDirection) throws UnknownHostException {
342 Match match = new Match();
343 List<Action> actions = new ArrayList<Action>();
345 if (flowDirection == LBConst.FORWARD_DIRECTION_LB_FLOW) {
346 match.setField(MatchType.DL_TYPE, EtherTypes.IPv4.shortValue());
347 match.setField(MatchType.NW_SRC, InetAddress.getByName(source.getIp()));
348 match.setField(MatchType.NW_DST, InetAddress.getByName(dest.getIp()));
349 match.setField(MatchType.NW_PROTO, IPProtocols.getProtocolNumberByte(dest.getProtocol()));
350 match.setField(MatchType.TP_SRC, source.getPort());
351 match.setField(MatchType.TP_DST, dest.getPort());
353 actions.add(new SetNwDst(InetAddress.getByName(destMachineIp)));
354 actions.add(new SetDlDst(destMachineMac));
357 if (flowDirection == LBConst.REVERSE_DIRECTION_LB_FLOW) {
358 match.setField(MatchType.DL_TYPE, EtherTypes.IPv4.shortValue());
359 match.setField(MatchType.NW_SRC, InetAddress.getByName(destMachineIp));
360 match.setField(MatchType.NW_DST, InetAddress.getByName(source.getIp()));
361 match.setField(MatchType.NW_PROTO, IPProtocols.getProtocolNumberByte(source.getProtocol()));
362 match.setField(MatchType.TP_SRC, dest.getPort());
363 match.setField(MatchType.TP_DST, source.getPort());
365 actions.add(new SetNwSrc(InetAddress.getByName(dest.getIp())));
366 actions.add(new SetDlSrc(destMachineMac));
369 actions.add(new Output(outport));
371 // Make sure the priority for IP switch entries is
372 // set to a level just above default drop entries
374 Flow flow = new Flow(match, actions);
375 flow.setIdleTimeout((short) 5);
376 flow.setHardTimeout((short) 0);
377 flow.setPriority(LB_IPSWITCH_PRIORITY);
379 String policyName = source.getIp() + ":" + source.getProtocol() + ":" + source.getPort();
380 String flowName = null;
382 if (flowDirection == LBConst.FORWARD_DIRECTION_LB_FLOW) {
383 flowName = "[" + policyName + ":" + source.getIp() + ":" + dest.getIp() + "]";
386 if (flowDirection == LBConst.REVERSE_DIRECTION_LB_FLOW) {
388 flowName = "[" + policyName + ":" + dest.getIp() + ":" + source.getIp() + "]";
391 FlowEntry fEntry = new FlowEntry(policyName, flowName, flow, sourceSwitch);
393 lbsLogger.trace("Install flow entry {} on node {}", fEntry.toString(), sourceSwitch.toString());
395 if (!this.ruleManager.checkFlowEntryConflict(fEntry)) {
396 if (this.ruleManager.installFlowEntry(fEntry).isSuccess()) {
399 lbsLogger.error("Error in installing flow entry to node : {}", sourceSwitch);
402 lbsLogger.error("Conflicting flow entry exists : {}", fEntry.toString());
408 * Function called by the dependency manager when all the required
409 * dependencies are satisfied
412 void init(Component c) {
413 Dictionary<?, ?> props = c.getServiceProperties();
415 this.containerName = (String) props.get("containerName");
417 lbsLogger.trace("Running container name:" + this.containerName);
420 // In the Global instance case the containerName is empty
421 this.containerName = "";
423 lbsLogger.trace(configManager.toString());
428 * Function called by the dependency manager when at least one dependency
429 * become unsatisfied or when the component is shutting down because for
430 * example bundle is being stopped.
437 * Function called by dependency manager after "init ()" is called and after
438 * the services provided by the class are registered in the service registry
445 * Function called by the dependency manager before the services exported by
446 * the component are unregistered, this will be followed by a "destroy ()"
454 * All the methods below are just proxy methods to direct the REST API
455 * requests to configuration manager. We need this redirection as currently,
456 * opendaylight supports only one implementation of the service.
459 public Set<VIP> getAllVIPs() {
460 return configManager.getAllVIPs();
464 public boolean vipExists(String name, String ip, String protocol, short protocolPort, String poolName) {
465 return configManager.vipExists(name, ip, protocol, protocolPort, poolName);
469 public boolean vipExists(VIP vip) {
470 return configManager.vipExists(vip);
474 public VIP createVIP(String name, String ip, String protocol, short protocolPort, String poolName) {
475 return configManager.createVIP(name, ip, protocol, protocolPort, poolName);
479 public VIP updateVIP(String name, String poolName) {
480 return configManager.updateVIP(name, poolName);
484 public VIP deleteVIP(String name) {
485 return configManager.deleteVIP(name);
489 public boolean memberExists(String name, String memberIP, String poolName) {
490 return configManager.memberExists(name, memberIP, poolName);
494 public Set<PoolMember> getAllPoolMembers(String poolName) {
496 return configManager.getAllPoolMembers(poolName);
500 public PoolMember addPoolMember(String name, String memberIP, String poolName) {
501 return configManager.addPoolMember(name, memberIP, poolName);
505 public PoolMember removePoolMember(String name, String poolName) {
507 return configManager.removePoolMember(name, poolName);
511 public Set<Pool> getAllPools() {
513 return configManager.getAllPools();
517 public Pool getPool(String poolName) {
518 return configManager.getPool(poolName);
522 public boolean poolExists(String name, String lbMethod) {
523 return configManager.poolExists(name, lbMethod);
527 public Pool createPool(String name, String lbMethod) {
528 return configManager.createPool(name, lbMethod);
532 public Pool deletePool(String poolName) {
533 return configManager.deletePool(poolName);
537 public boolean vipExists(String name) {
538 return configManager.vipExists(name);
542 public boolean memberExists(String name, String poolName) {
543 return configManager.memberExists(name, poolName);
547 public boolean poolExists(String name) {
548 return configManager.poolExists(name);
552 public String getVIPAttachedPool(String name) {
553 return configManager.getVIPAttachedPool(name);