Preliminary performance testing shows that using Epoll on Linux x64
increases Map-Request, un-authenticated Map-Register and authenticated
Map-Register response rates from an average of 57k, 44k, and 31k replies
per second respectively to 88k, 55k, and 38k replies per second on our
internal reference testing platform.
Note that the sending rate is 100k requests per second, which means that
at higher sending rates the Map-Request performance may be even better
(although still "lossy").
This is the first step in improving southbound performance, with a
second patch to follow for multi-threaded packet handling, which depends
on Epoll (thus this patch).
Change-Id: I540a755850fb4c40ed0b1a4c745dbb4a5fa7bc9f
Signed-off-by: Lorand Jakab <lojakab@cisco.com>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ <classifier>linux-x86_64</classifier>
+ </dependency>
<dependency>
<groupId>junit-addons</groupId>
<artifactId>junit-addons</artifactId>
<dependency>
<groupId>junit-addons</groupId>
<artifactId>junit-addons</artifactId>
import static io.netty.buffer.Unpooled.wrappedBuffer;
import com.google.common.base.Preconditions;
import static io.netty.buffer.Unpooled.wrappedBuffer;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil;
+import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.EventLoopGroup;
+import io.netty.channel.epoll.Epoll;
+import io.netty.channel.epoll.EpollDatagramChannel;
+import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.DatagramPacket;
import io.netty.channel.socket.nio.NioDatagramChannel;
import io.netty.util.concurrent.DefaultThreadFactory;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.DatagramPacket;
import io.netty.channel.socket.nio.NioDatagramChannel;
import io.netty.util.concurrent.DefaultThreadFactory;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.concurrent.ThreadFactory;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.concurrent.ThreadFactory;
import org.opendaylight.controller.md.sal.binding.api.DataBroker;
import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
import org.opendaylight.lispflowmapping.lisp.type.LispMessage;
import org.opendaylight.controller.md.sal.binding.api.DataBroker;
import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
import org.opendaylight.lispflowmapping.lisp.type.LispMessage;
private LispSouthboundHandler lispSouthboundHandler;
private LispXtrSouthboundHandler lispXtrSouthboundHandler;
private NotificationPublishService notificationPublishService;
private LispSouthboundHandler lispSouthboundHandler;
private LispXtrSouthboundHandler lispXtrSouthboundHandler;
private NotificationPublishService notificationPublishService;
- private NioDatagramChannel channel;
+ private Channel channel;
+ private Channel xtrChannel;
+ private Class channelType;
private volatile int xtrPort = LispMessage.XTR_PORT_NUM;
private volatile boolean listenOnXtrPort = false;
private volatile int xtrPort = LispMessage.XTR_PORT_NUM;
private volatile boolean listenOnXtrPort = false;
- private NioDatagramChannel xtrChannel;
private LispSouthboundStats statistics = new LispSouthboundStats();
private Bootstrap bootstrap = new Bootstrap();
private Bootstrap xtrBootstrap = new Bootstrap();
private ThreadFactory threadFactory = new DefaultThreadFactory("lisp-sb");
private LispSouthboundStats statistics = new LispSouthboundStats();
private Bootstrap bootstrap = new Bootstrap();
private Bootstrap xtrBootstrap = new Bootstrap();
private ThreadFactory threadFactory = new DefaultThreadFactory("lisp-sb");
- private EventLoopGroup eventLoopGroup = new NioEventLoopGroup(0, threadFactory);
+ private EventLoopGroup eventLoopGroup;
private DataBroker dataBroker;
public LispSouthboundPlugin(final DataBroker dataBroker,
private DataBroker dataBroker;
public LispSouthboundPlugin(final DataBroker dataBroker,
lispXtrSouthboundHandler = new LispXtrSouthboundHandler();
lispXtrSouthboundHandler.setNotificationProvider(this.notificationPublishService);
lispXtrSouthboundHandler = new LispXtrSouthboundHandler();
lispXtrSouthboundHandler.setNotificationProvider(this.notificationPublishService);
+ if (Epoll.isAvailable()) {
+ eventLoopGroup = new EpollEventLoopGroup(0, threadFactory);
+ channelType = EpollDatagramChannel.class;
+ LOG.debug("Using Netty Epoll for UDP sockets");
+ } else {
+ eventLoopGroup = new NioEventLoopGroup(0, threadFactory);
+ channelType = NioDatagramChannel.class;
+ LOG.debug("Using Netty I/O (non-Epoll) for UDP sockets");
+ }
+
bootstrap.group(eventLoopGroup);
bootstrap.group(eventLoopGroup);
- bootstrap.channel(NioDatagramChannel.class);
+ bootstrap.channel(channelType);
bootstrap.handler(lispSouthboundHandler);
xtrBootstrap.group(eventLoopGroup);
bootstrap.handler(lispSouthboundHandler);
xtrBootstrap.group(eventLoopGroup);
- xtrBootstrap.channel(NioDatagramChannel.class);
+ xtrBootstrap.channel(channelType);
xtrBootstrap.handler(lispXtrSouthboundHandler);
start();
xtrBootstrap.handler(lispXtrSouthboundHandler);
start();
@SuppressWarnings("checkstyle:IllegalCatch")
private void start() {
try {
@SuppressWarnings("checkstyle:IllegalCatch")
private void start() {
try {
- channel = (NioDatagramChannel) bootstrap.bind(bindingAddress, LispMessage.PORT_NUM).sync().channel();
+ channel = bootstrap.bind(bindingAddress, LispMessage.PORT_NUM).sync().channel();
LOG.debug("Binding LISP UDP listening socket to {}:{}", bindingAddress, LispMessage.PORT_NUM);
} catch (Exception e) {
LOG.error("Failed to open main socket ", e);
LOG.debug("Binding LISP UDP listening socket to {}:{}", bindingAddress, LispMessage.PORT_NUM);
} catch (Exception e) {
LOG.error("Failed to open main socket ", e);
private void startXtr() {
if (listenOnXtrPort) {
try {
private void startXtr() {
if (listenOnXtrPort) {
try {
- xtrChannel = (NioDatagramChannel) xtrBootstrap.bind(bindingAddress, xtrPort).sync().channel();
+ xtrChannel = xtrBootstrap.bind(bindingAddress, xtrPort).sync().channel();
LOG.debug("Binding LISP xTR UDP listening socket to {}:{}", bindingAddress, xtrPort);
} catch (Exception e) {
LOG.error("Failed to open xTR socket ", e);
LOG.debug("Binding LISP xTR UDP listening socket to {}:{}", bindingAddress, xtrPort);
} catch (Exception e) {
LOG.error("Failed to open xTR socket ", e);