/* * Copyright IBM Corporation, 2013. All rights reserved. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v1.0 which accompanies this distribution, * and is available at http://www.eclipse.org/legal/epl-v10.html */ package org.opendaylight.controller.samples.loadbalancer.internal; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Dictionary; import java.util.List; import java.util.Set; import org.apache.felix.dm.Component; import org.opendaylight.controller.forwardingrulesmanager.FlowEntry; import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager; import org.opendaylight.controller.hosttracker.HostIdFactory; import org.opendaylight.controller.hosttracker.IHostId; import org.opendaylight.controller.hosttracker.IfIptoHost; import org.opendaylight.controller.hosttracker.hostAware.HostNodeConnector; import org.opendaylight.controller.sal.action.Action; import org.opendaylight.controller.sal.action.Output; import org.opendaylight.controller.sal.action.SetDlDst; import org.opendaylight.controller.sal.action.SetDlSrc; import org.opendaylight.controller.sal.action.SetNwDst; import org.opendaylight.controller.sal.action.SetNwSrc; import org.opendaylight.controller.sal.core.Node; import org.opendaylight.controller.sal.core.NodeConnector; import org.opendaylight.controller.sal.core.Path; import org.opendaylight.controller.sal.flowprogrammer.Flow; import org.opendaylight.controller.sal.match.Match; import org.opendaylight.controller.sal.match.MatchType; import org.opendaylight.controller.sal.packet.Ethernet; import org.opendaylight.controller.sal.packet.IDataPacketService; import org.opendaylight.controller.sal.packet.IListenDataPacket; import org.opendaylight.controller.sal.packet.IPv4; import org.opendaylight.controller.sal.packet.Packet; import org.opendaylight.controller.sal.packet.PacketResult; import org.opendaylight.controller.sal.packet.RawPacket; import org.opendaylight.controller.sal.routing.IRouting; import org.opendaylight.controller.sal.utils.EtherTypes; import org.opendaylight.controller.sal.utils.GlobalConstants; import org.opendaylight.controller.sal.utils.IPProtocols; import org.opendaylight.controller.samples.loadbalancer.ConfigManager; import org.opendaylight.controller.samples.loadbalancer.IConfigManager; import org.opendaylight.controller.samples.loadbalancer.LBConst; import org.opendaylight.controller.samples.loadbalancer.LBUtil; import org.opendaylight.controller.samples.loadbalancer.entities.Client; import org.opendaylight.controller.samples.loadbalancer.entities.Pool; import org.opendaylight.controller.samples.loadbalancer.entities.PoolMember; import org.opendaylight.controller.samples.loadbalancer.entities.VIP; import org.opendaylight.controller.samples.loadbalancer.policies.RandomLBPolicy; import org.opendaylight.controller.samples.loadbalancer.policies.RoundRobinLBPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This class is the main class that represents the load balancer service. This * is a sample load balancer application that balances traffic to backend * servers based on the source address and source port on each incoming packet. * The service reactively installs OpenFlow rules to direct all packets with a * specific source address and source port to one of the appropriate backend * servers. The servers may be chosen using a round robin policy or a random * policy. This service can be configured via a REST APIs which are similar to * the OpenStack Quantum LBaaS (Load-balancer-as-a-Service) v1.0 API proposal * (http://wiki.openstack.org/Quantum/LBaaS) * * To use this service, a virtual IP (or VIP) should be exposed to the clients * of this service and used as the destination address. A VIP is a entity that * comprises of a virtual IP, port and protocol (TCP or UDP). Assumptions: 1. * One or more VIPs may be mapped to the same server pool. All VIPs that share * the same pool must also share the same load balancing policy (random or round * robin). * * 2. Only one server pool can be be assigned to a VIP. * * 3. All flow rules are installed with an idle timeout of 5 seconds. * * 4. Packets to a VIP must leave the OpenFlow cluster from the same switch from * where it entered it. * * 5. When you delete a VIP or a server pool or a server from a pool, the * service does not delete the flow rules it has already installed. The flow * rules should automatically time out after the idle timeout of 5 seconds. * */ public class LoadBalancerService implements IListenDataPacket, IConfigManager { /* * Logger instance */ private static Logger lbsLogger = LoggerFactory.getLogger(LoadBalancerService.class); /* * Single instance of the configuration manager. Application passes this * reference to all the new policies implemented for load balancing. */ private static ConfigManager configManager = new ConfigManager(); /* * Round robing policy instance. Need to implement factory patterns to get * policy instance. */ private static RoundRobinLBPolicy rrLBMethod = new RoundRobinLBPolicy(configManager); /* * Random policy instance. */ private static RandomLBPolicy ranLBMethod = new RandomLBPolicy(configManager); /* * Reference to the data packet service */ private IDataPacketService dataPacketService = null; /* * Reference to the host tracker service */ private IfIptoHost hostTracker; /* * Reference to the forwarding manager */ private IForwardingRulesManager ruleManager; /* * Reference to the routing service */ private IRouting routing; /* * Load balancer application installs all flows with priority 2. */ private static short LB_IPSWITCH_PRIORITY = 2; /* * Name of the container where this application will register. */ private String containerName = null; /* * Set/unset methods for the service instance that load balancer service * requires */ public String getContainerName() { if (containerName == null) return GlobalConstants.DEFAULT.toString(); return containerName; } void setDataPacketService(IDataPacketService s) { this.dataPacketService = s; } void unsetDataPacketService(IDataPacketService s) { if (this.dataPacketService == s) { this.dataPacketService = null; } } public void setRouting(IRouting routing) { this.routing = routing; } public void unsetRouting(IRouting routing) { if (this.routing == routing) { this.routing = null; } } public void setHostTracker(IfIptoHost hostTracker) { lbsLogger.debug("Setting HostTracker"); this.hostTracker = hostTracker; } public void unsetHostTracker(IfIptoHost hostTracker) { if (this.hostTracker == hostTracker) { this.hostTracker = null; } } public void setForwardingRulesManager(IForwardingRulesManager forwardingRulesManager) { lbsLogger.debug("Setting ForwardingRulesManager"); this.ruleManager = forwardingRulesManager; } public void unsetForwardingRulesManager(IForwardingRulesManager forwardingRulesManager) { if (this.ruleManager == forwardingRulesManager) { this.ruleManager = null; } } /** * This method receives first packet of flows for which there is no matching * flow rule installed on the switch. IP addresses used for VIPs are not * supposed to be used by any real/virtual host in the network. Hence, any * forwarding/routing service will not install any flows rules matching * these VIPs. This ensures that all the flows destined for VIPs will not * find a match in the switch and will be forwarded to the load balancing * service. Service will decide where to route this traffic based on the * load balancing policy of the VIP's attached pool and will install * appropriate flow rules in a reactive manner. */ @Override public PacketResult receiveDataPacket(RawPacket inPkt) { if (inPkt == null) { return PacketResult.IGNORED; } Packet formattedPak = this.dataPacketService.decodeDataPacket(inPkt); if (formattedPak instanceof Ethernet) { byte[] vipMacAddr = ((Ethernet) formattedPak).getDestinationMACAddress(); Object ipPkt = formattedPak.getPayload(); if (ipPkt instanceof IPv4) { lbsLogger.debug("Packet recieved from switch : {}", inPkt.getIncomingNodeConnector().getNode() .toString()); IPv4 ipv4Pkt = (IPv4) ipPkt; if (IPProtocols.getProtocolName(ipv4Pkt.getProtocol()).equals(IPProtocols.TCP.toString()) || IPProtocols.getProtocolName(ipv4Pkt.getProtocol()).equals(IPProtocols.UDP.toString())) { lbsLogger.debug("Packet protocol : {}", IPProtocols.getProtocolName(ipv4Pkt.getProtocol())); Client client = new LBUtil().getClientFromPacket(ipv4Pkt); VIP vip = new LBUtil().getVIPFromPacket(ipv4Pkt); if (configManager.vipExists(vip)) { VIP vipWithPoolName = configManager.getVIPWithPoolName(vip); String poolMemberIp = null; if (vipWithPoolName.getPoolName() == null) { lbsLogger.error("No pool attached. Please attach pool with the VIP -- {}", vip); return PacketResult.IGNORED; } if (configManager.getPool(vipWithPoolName.getPoolName()).getLbMethod() .equalsIgnoreCase(LBConst.ROUND_ROBIN_LB_METHOD)) { poolMemberIp = rrLBMethod.getPoolMemberForClient(client, vipWithPoolName); } if (configManager.getPool(vipWithPoolName.getPoolName()).getLbMethod() .equalsIgnoreCase(LBConst.RANDOM_LB_METHOD)) { poolMemberIp = ranLBMethod.getPoolMemberForClient(client, vipWithPoolName); } try { Node clientNode = inPkt.getIncomingNodeConnector().getNode(); // HostTracker hosts db key scheme implementation IHostId id = HostIdFactory.create(InetAddress.getByName(poolMemberIp), null); HostNodeConnector hnConnector = this.hostTracker.hostFind(id); Node destNode = hnConnector.getnodeconnectorNode(); lbsLogger.debug("Client is connected to switch : {}", clientNode.toString()); lbsLogger .debug("Destination pool machine is connected to switch : {}", destNode.toString()); // Get path between both the nodes NodeConnector forwardPort = null; if (clientNode.getNodeIDString().equals(destNode.getNodeIDString())) { forwardPort = hnConnector.getnodeConnector(); lbsLogger .trace("Both source (client) and destination pool machine is connected to same switch nodes. Respective ports are - {},{}", forwardPort, inPkt.getIncomingNodeConnector()); } else { Path route = this.routing.getRoute(clientNode, destNode); lbsLogger.trace("Path between source (client) and destination switch nodes : {}", route.toString()); forwardPort = route.getEdges().get(0).getTailNodeConnector(); } if (installLoadBalancerFlow(client, vip, clientNode, poolMemberIp, hnConnector.getDataLayerAddressBytes(), forwardPort, LBConst.FORWARD_DIRECTION_LB_FLOW)) { lbsLogger.trace("Traffic from client : {} will be routed " + "to pool machine : {}", client, poolMemberIp); } else { lbsLogger.error("Not able to route traffic from client : {}", client); } if (installLoadBalancerFlow(client, vip, clientNode, poolMemberIp, vipMacAddr, inPkt.getIncomingNodeConnector(), LBConst.REVERSE_DIRECTION_LB_FLOW)) { lbsLogger.trace("Flow rule installed to change the source ip/mac from " + "pool machine ip {} to VIP {} for traffic coming pool machine", poolMemberIp, vip); } else { lbsLogger.error("Not able to route traffic from client : {}", client); } } catch (UnknownHostException e) { lbsLogger.error("Pool member not found in the network : {}", e.getMessage()); lbsLogger.error("", e); } } } } } return PacketResult.IGNORED; } /* * This method installs the flow rule for routing the traffic between two * hosts. * * @param source Traffic is sent by this source * * @param dest Traffic is destined to this destination (VIP) * * @param sourceSwitch Switch from where controller received the packet * * @param destMachineIp IP address of the pool member where traffic needs to * be routed * * @param destMachineMac MAC address of the pool member where traffic needs * to be routed * * @param outport Use this port to send out traffic * * @param flowDirection FORWARD_DIRECTION_LB_FLOW or * REVERSE_DIRECTION_LB_FLOW * * @return true If flow installation was successful false else * * @throws UnknownHostException */ private boolean installLoadBalancerFlow(Client source, VIP dest, Node sourceSwitch, String destMachineIp, byte[] destMachineMac, NodeConnector outport, int flowDirection) throws UnknownHostException { Match match = new Match(); List actions = new ArrayList(); if (flowDirection == LBConst.FORWARD_DIRECTION_LB_FLOW) { match.setField(MatchType.DL_TYPE, EtherTypes.IPv4.shortValue()); match.setField(MatchType.NW_SRC, InetAddress.getByName(source.getIp())); match.setField(MatchType.NW_DST, InetAddress.getByName(dest.getIp())); match.setField(MatchType.NW_PROTO, IPProtocols.getProtocolNumberByte(dest.getProtocol())); match.setField(MatchType.TP_SRC, source.getPort()); match.setField(MatchType.TP_DST, dest.getPort()); actions.add(new SetNwDst(InetAddress.getByName(destMachineIp))); actions.add(new SetDlDst(destMachineMac)); } if (flowDirection == LBConst.REVERSE_DIRECTION_LB_FLOW) { match.setField(MatchType.DL_TYPE, EtherTypes.IPv4.shortValue()); match.setField(MatchType.NW_SRC, InetAddress.getByName(destMachineIp)); match.setField(MatchType.NW_DST, InetAddress.getByName(source.getIp())); match.setField(MatchType.NW_PROTO, IPProtocols.getProtocolNumberByte(source.getProtocol())); match.setField(MatchType.TP_SRC, dest.getPort()); match.setField(MatchType.TP_DST, source.getPort()); actions.add(new SetNwSrc(InetAddress.getByName(dest.getIp()))); actions.add(new SetDlSrc(destMachineMac)); } actions.add(new Output(outport)); // Make sure the priority for IP switch entries is // set to a level just above default drop entries Flow flow = new Flow(match, actions); flow.setIdleTimeout((short) 5); flow.setHardTimeout((short) 0); flow.setPriority(LB_IPSWITCH_PRIORITY); String policyName = source.getIp() + ":" + source.getProtocol() + ":" + source.getPort(); String flowName = null; if (flowDirection == LBConst.FORWARD_DIRECTION_LB_FLOW) { flowName = "[" + policyName + ":" + source.getIp() + ":" + dest.getIp() + "]"; } if (flowDirection == LBConst.REVERSE_DIRECTION_LB_FLOW) { flowName = "[" + policyName + ":" + dest.getIp() + ":" + source.getIp() + "]"; } FlowEntry fEntry = new FlowEntry(policyName, flowName, flow, sourceSwitch); lbsLogger.trace("Install flow entry {} on node {}", fEntry.toString(), sourceSwitch.toString()); if (!this.ruleManager.checkFlowEntryConflict(fEntry)) { if (this.ruleManager.installFlowEntry(fEntry).isSuccess()) { return true; } else { lbsLogger.error("Error in installing flow entry to node : {}", sourceSwitch); } } else { lbsLogger.error("Conflicting flow entry exists : {}", fEntry.toString()); } return false; } /** * Function called by the dependency manager when all the required * dependencies are satisfied * */ void init(Component c) { Dictionary props = c.getServiceProperties(); if (props != null) { this.containerName = (String) props.get("containerName"); lbsLogger.trace("Running container name:" + this.containerName); } else { // In the Global instance case the containerName is empty this.containerName = ""; } lbsLogger.trace(configManager.toString()); } /** * Function called by the dependency manager when at least one dependency * become unsatisfied or when the component is shutting down because for * example bundle is being stopped. * */ void destroy() { } /** * Function called by dependency manager after "init ()" is called and after * the services provided by the class are registered in the service registry * */ void start() { } /** * Function called by the dependency manager before the services exported by * the component are unregistered, this will be followed by a "destroy ()" * calls * */ void stop() { } /* * All the methods below are just proxy methods to direct the REST API * requests to configuration manager. We need this redirection as currently, * opendaylight supports only one implementation of the service. */ @Override public Set getAllVIPs() { return configManager.getAllVIPs(); } @Override public boolean vipExists(String name, String ip, String protocol, short protocolPort, String poolName) { return configManager.vipExists(name, ip, protocol, protocolPort, poolName); } @Override public boolean vipExists(VIP vip) { return configManager.vipExists(vip); } @Override public VIP createVIP(String name, String ip, String protocol, short protocolPort, String poolName) { return configManager.createVIP(name, ip, protocol, protocolPort, poolName); } @Override public VIP updateVIP(String name, String poolName) { return configManager.updateVIP(name, poolName); } @Override public VIP deleteVIP(String name) { return configManager.deleteVIP(name); } @Override public boolean memberExists(String name, String memberIP, String poolName) { return configManager.memberExists(name, memberIP, poolName); } @Override public Set getAllPoolMembers(String poolName) { return configManager.getAllPoolMembers(poolName); } @Override public PoolMember addPoolMember(String name, String memberIP, String poolName) { return configManager.addPoolMember(name, memberIP, poolName); } @Override public PoolMember removePoolMember(String name, String poolName) { return configManager.removePoolMember(name, poolName); } @Override public Set getAllPools() { return configManager.getAllPools(); } @Override public Pool getPool(String poolName) { return configManager.getPool(poolName); } @Override public boolean poolExists(String name, String lbMethod) { return configManager.poolExists(name, lbMethod); } @Override public Pool createPool(String name, String lbMethod) { return configManager.createPool(name, lbMethod); } @Override public Pool deletePool(String poolName) { return configManager.deletePool(poolName); } @Override public boolean vipExists(String name) { return configManager.vipExists(name); } @Override public boolean memberExists(String name, String poolName) { return configManager.memberExists(name, poolName); } @Override public boolean poolExists(String name) { return configManager.poolExists(name); } @Override public String getVIPAttachedPool(String name) { return configManager.getVIPAttachedPool(name); } }