2 * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.protocol.bgp.rib.impl;
10 import static java.util.Objects.requireNonNull;
12 import com.google.common.annotations.VisibleForTesting;
13 import com.google.common.base.MoreObjects;
14 import com.google.common.base.MoreObjects.ToStringHelper;
15 import com.google.common.collect.ImmutableSet;
16 import io.netty.buffer.ByteBufUtil;
17 import io.netty.channel.Channel;
18 import io.netty.channel.ChannelFuture;
19 import io.netty.channel.ChannelFutureListener;
20 import io.netty.channel.ChannelHandlerContext;
21 import io.netty.channel.SimpleChannelInboundHandler;
22 import io.netty.util.concurrent.ScheduledFuture;
23 import java.io.IOException;
24 import java.nio.channels.NonWritableChannelException;
25 import java.util.ArrayList;
26 import java.util.Date;
27 import java.util.HashSet;
28 import java.util.List;
29 import java.util.Optional;
31 import java.util.concurrent.TimeUnit;
32 import java.util.function.Function;
33 import org.checkerframework.checker.lock.qual.GuardedBy;
34 import org.checkerframework.checker.lock.qual.Holding;
35 import org.opendaylight.protocol.bgp.parser.AsNumberUtil;
36 import org.opendaylight.protocol.bgp.parser.BGPDocumentedException;
37 import org.opendaylight.protocol.bgp.parser.BGPError;
38 import org.opendaylight.protocol.bgp.parser.BgpExtendedMessageUtil;
39 import org.opendaylight.protocol.bgp.parser.BgpTableTypeImpl;
40 import org.opendaylight.protocol.bgp.parser.GracefulRestartUtil;
41 import org.opendaylight.protocol.bgp.parser.spi.MultiPathSupport;
42 import org.opendaylight.protocol.bgp.parser.spi.PeerConstraint;
43 import org.opendaylight.protocol.bgp.parser.spi.pojo.MultiPathSupportImpl;
44 import org.opendaylight.protocol.bgp.rib.impl.spi.BGPMessagesListener;
45 import org.opendaylight.protocol.bgp.rib.impl.spi.BGPPeerRegistry;
46 import org.opendaylight.protocol.bgp.rib.impl.spi.BGPSessionPreferences;
47 import org.opendaylight.protocol.bgp.rib.impl.state.BGPSessionStateImpl;
48 import org.opendaylight.protocol.bgp.rib.impl.state.BGPSessionStateProvider;
49 import org.opendaylight.protocol.bgp.rib.spi.BGPSession;
50 import org.opendaylight.protocol.bgp.rib.spi.BGPSessionListener;
51 import org.opendaylight.protocol.bgp.rib.spi.BGPTerminationReason;
52 import org.opendaylight.protocol.bgp.rib.spi.State;
53 import org.opendaylight.protocol.bgp.rib.spi.state.BGPSessionState;
54 import org.opendaylight.protocol.bgp.rib.spi.state.BGPTimersState;
55 import org.opendaylight.protocol.bgp.rib.spi.state.BGPTransportState;
56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.AsNumber;
57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.message.rev200120.Keepalive;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.message.rev200120.KeepaliveBuilder;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.message.rev200120.Notify;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.message.rev200120.NotifyBuilder;
62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.message.rev200120.Open;
63 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.message.rev200120.Update;
64 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.message.rev200120.open.message.BgpParameters;
65 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.message.rev200120.open.message.bgp.parameters.OptionalCapabilities;
66 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.message.rev200120.open.message.bgp.parameters.optional.capabilities.CParameters;
67 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.BgpTableType;
68 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.CParameters1;
69 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.MpCapabilities;
70 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.RouteRefresh;
71 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.mp.capabilities.AddPathCapability;
72 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.mp.capabilities.GracefulRestartCapability;
73 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.mp.capabilities.LlGracefulRestartCapability;
74 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.mp.capabilities.MultiprotocolCapability;
75 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.mp.capabilities.add.path.capability.AddressFamilies;
76 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.rib.rev180329.rib.TablesKey;
77 import org.opendaylight.yangtools.yang.binding.ChildOf;
78 import org.opendaylight.yangtools.yang.binding.Notification;
79 import org.opendaylight.yangtools.yang.common.Uint8;
80 import org.slf4j.Logger;
81 import org.slf4j.LoggerFactory;
84 public class BGPSessionImpl extends SimpleChannelInboundHandler<Notification> implements BGPSession,
85 BGPSessionStateProvider, AutoCloseable {
87 private static final Logger LOG = LoggerFactory.getLogger(BGPSessionImpl.class);
89 private static final Notification KEEP_ALIVE = new KeepaliveBuilder().build();
91 private static final int KA_TO_DEADTIMER_RATIO = 3;
93 static final String END_OF_INPUT = "End of input detected. Close the session.";
96 * System.nanoTime value about when was sent the last message.
99 private long lastMessageSentAt;
102 * System.nanoTime value about when was received the last message.
104 private long lastMessageReceivedAt;
106 private final BGPSessionListener listener;
108 private final BGPSynchronization sync;
110 private int kaCounter = 0;
112 private final Channel channel;
115 private State state = State.OPEN_CONFIRM;
117 private final Set<BgpTableType> tableTypes;
118 private final List<AddressFamilies> addPathTypes;
119 private final long holdTimerNanos;
120 private final long keepAliveNanos;
121 private final AsNumber asNumber;
122 private final Ipv4Address bgpId;
123 private final BGPPeerRegistry peerRegistry;
124 private final ChannelOutputLimiter limiter;
125 private final BGPSessionStateImpl sessionState;
126 private final GracefulRestartCapability gracefulCapability;
127 private final LlGracefulRestartCapability llGracefulCapability;
128 private boolean terminationReasonNotified;
130 public BGPSessionImpl(final BGPSessionListener listener, final Channel channel, final Open remoteOpen,
131 final BGPSessionPreferences localPreferences, final BGPPeerRegistry peerRegistry) {
132 this(listener, channel, remoteOpen, localPreferences.getHoldTime(), peerRegistry);
135 public BGPSessionImpl(final BGPSessionListener listener, final Channel channel, final Open remoteOpen,
136 final int localHoldTimer, final BGPPeerRegistry peerRegistry) {
137 this.listener = requireNonNull(listener);
138 this.channel = requireNonNull(channel);
139 this.limiter = new ChannelOutputLimiter(this);
140 this.channel.pipeline().addLast(this.limiter);
142 final int remoteHoldTimer = remoteOpen.getHoldTimer().toJava();
143 final int holdTimerValue = Math.min(remoteHoldTimer, localHoldTimer);
144 LOG.info("BGP HoldTimer new value: {}", holdTimerValue);
145 this.holdTimerNanos = TimeUnit.SECONDS.toNanos(holdTimerValue);
146 this.keepAliveNanos = TimeUnit.SECONDS.toNanos(holdTimerValue / KA_TO_DEADTIMER_RATIO);
148 this.asNumber = AsNumberUtil.advertizedAsNumber(remoteOpen);
149 this.peerRegistry = peerRegistry;
150 this.sessionState = new BGPSessionStateImpl();
152 final Set<TablesKey> tts = new HashSet<>();
153 final Set<BgpTableType> tats = new HashSet<>();
154 final List<AddressFamilies> addPathCapabilitiesList = new ArrayList<>();
155 final List<BgpParameters> bgpParameters = remoteOpen.getBgpParameters();
156 if (bgpParameters != null) {
157 for (final BgpParameters param : bgpParameters) {
158 for (final OptionalCapabilities optCapa : param.nonnullOptionalCapabilities()) {
159 final CParameters cParam = optCapa.getCParameters();
160 final CParameters1 cParam1 = cParam.augmentation(CParameters1.class);
161 if (cParam1 != null) {
162 final MultiprotocolCapability multi = cParam1.getMultiprotocolCapability();
164 final TablesKey tt = new TablesKey(multi.getAfi(), multi.getSafi());
165 LOG.trace("Added table type to sync {}", tt);
167 tats.add(new BgpTableTypeImpl(tt.getAfi(), tt.getSafi()));
169 final AddPathCapability addPathCap = cParam1.getAddPathCapability();
170 if (addPathCap != null) {
171 addPathCapabilitiesList.addAll(addPathCap.getAddressFamilies());
177 this.gracefulCapability = findSingleCapability(bgpParameters, "Graceful Restart",
178 CParameters1::getGracefulRestartCapability).orElse(GracefulRestartUtil.EMPTY_GR_CAPABILITY);
179 this.llGracefulCapability = findSingleCapability(bgpParameters, "Long-lived Graceful Restart",
180 CParameters1::getLlGracefulRestartCapability).orElse(GracefulRestartUtil.EMPTY_LLGR_CAPABILITY);
182 this.gracefulCapability = GracefulRestartUtil.EMPTY_GR_CAPABILITY;
183 this.llGracefulCapability = GracefulRestartUtil.EMPTY_LLGR_CAPABILITY;
186 this.sync = new BGPSynchronization(this.listener, tts);
187 this.tableTypes = tats;
188 this.addPathTypes = addPathCapabilitiesList;
190 if (!this.addPathTypes.isEmpty()) {
191 addDecoderConstraint(MultiPathSupport.class,
192 MultiPathSupportImpl.createParserMultiPathSupport(this.addPathTypes));
195 if (holdTimerValue != 0) {
196 channel.eventLoop().schedule(this::handleHoldTimer, this.holdTimerNanos, TimeUnit.NANOSECONDS);
197 channel.eventLoop().schedule(this::handleKeepaliveTimer, this.keepAliveNanos, TimeUnit.NANOSECONDS);
199 this.bgpId = remoteOpen.getBgpIdentifier();
200 this.sessionState.advertizeCapabilities(holdTimerValue, channel.remoteAddress(), channel.localAddress(),
201 this.tableTypes, bgpParameters);
204 private static <T extends ChildOf<MpCapabilities>> Optional<T> findSingleCapability(
205 final List<BgpParameters> bgpParameters, final String name, final Function<CParameters1, T> extractor) {
206 final List<T> found = new ArrayList<>(1);
207 for (BgpParameters bgpParams : bgpParameters) {
208 for (OptionalCapabilities optCapability : bgpParams.nonnullOptionalCapabilities()) {
209 final CParameters cparam = optCapability.getCParameters();
210 if (cparam != null) {
211 final CParameters1 augment = cparam.augmentation(CParameters1.class);
212 if (augment != null) {
213 final T capa = extractor.apply(augment);
222 final Set<T> set = ImmutableSet.copyOf(found);
223 switch (set.size()) {
225 LOG.debug("{} capability not advertised.", name);
226 return Optional.empty();
228 return Optional.of(found.get(0));
230 LOG.warn("Multiple instances of {} capability advertised: {}, ignoring.", name, set);
231 return Optional.empty();
236 * Set the extend message coder for current channel.
237 * The reason for separating this part from constructor is, in #channel.pipeline().replace(..), the
238 * invokeChannelRead() will be invoked after the original message coder handler got removed. And there
239 * is chance that before the session instance is fully initiated (constructor returns), a KeepAlive
240 * message arrived already in the channel buffer. Thus #AbstractBGPSessionNegotiator.handleMessage(..)
241 * gets invoked again and a deadlock is caused. A BGP final state machine error will happen as BGP
242 * negotiator is still in OPEN_SENT state as the session constructor hasn't returned yet.
244 public synchronized void setChannelExtMsgCoder(final Open remoteOpen) {
245 final boolean enableExMess = BgpExtendedMessageUtil.advertizedBgpExtendedMessageCapability(remoteOpen);
247 BGPMessageHeaderDecoder.enableExtendedMessages(this.channel);
252 public synchronized void close() {
253 if (this.state != State.IDLE) {
254 if (!this.terminationReasonNotified) {
255 this.writeAndFlush(new NotifyBuilder().setErrorCode(BGPError.CEASE.getCode())
256 .setErrorSubcode(BGPError.CEASE.getSubcode()).build());
258 this.closeWithoutMessage();
263 * Handles incoming message based on their type.
265 * @param msg incoming message
267 void handleMessage(final Notification msg) {
268 // synchronize on listener and then on this object to ensure correct order of locking
269 synchronized (this.listener) {
270 synchronized (this) {
271 if (this.state == State.IDLE) {
275 // Update last reception time
276 this.lastMessageReceivedAt = System.nanoTime();
278 if (msg instanceof Open) {
279 // Open messages should not be present here
280 terminate(new BGPDocumentedException(null, BGPError.FSM_ERROR));
281 } else if (msg instanceof Notify) {
282 final Notify notify = (Notify) msg;
283 // Notifications are handled internally
284 LOG.info("Session closed because Notification message received: {} / {}, data={}",
285 notify.getErrorCode(),
286 notify.getErrorSubcode(),
287 notify.getData() != null ? ByteBufUtil.hexDump(notify.getData()) : null);
288 notifyTerminationReasonAndCloseWithoutMessage(notify.getErrorCode(), notify.getErrorSubcode());
289 } else if (msg instanceof Keepalive) {
290 // Keepalives are handled internally
291 LOG.trace("Received KeepAlive message.");
293 if (this.kaCounter >= 2) {
294 this.sync.kaReceived();
296 } else if (msg instanceof RouteRefresh) {
297 this.listener.onMessage(this, msg);
298 } else if (msg instanceof Update) {
299 this.listener.onMessage(this, msg);
300 this.sync.updReceived((Update) msg);
302 LOG.warn("Ignoring unhandled message: {}.", msg.getClass());
305 this.sessionState.messageReceived(msg);
306 } catch (final BGPDocumentedException e) {
313 @Holding({"this.listener", "this"})
314 private void notifyTerminationReasonAndCloseWithoutMessage(final BGPError error) {
315 this.terminationReasonNotified = true;
316 this.closeWithoutMessage();
317 this.listener.onSessionTerminated(this, new BGPTerminationReason(error));
320 @Holding({"this.listener", "this"})
321 private void notifyTerminationReasonAndCloseWithoutMessage(final Uint8 errorCode, final Uint8 errorSubcode) {
322 this.terminationReasonNotified = true;
323 this.closeWithoutMessage();
324 this.listener.onSessionTerminated(this, new BGPTerminationReason(BGPError.forValue(errorCode, errorSubcode)));
328 // synchronize on listener and then on this object to ensure correct order of locking
329 synchronized (this.listener) {
330 synchronized (this) {
331 if (this.state == State.UP) {
332 LOG.info(END_OF_INPUT);
333 this.listener.onSessionDown(this, new IOException(END_OF_INPUT));
340 private ChannelFuture writeEpilogue(final ChannelFuture future, final Notification msg) {
341 future.addListener((ChannelFutureListener) f -> {
343 LOG.trace("Message {} sent to socket {}", msg, this.channel);
345 LOG.warn("Failed to send message {} to socket {}", msg, this.channel, f.cause());
348 this.lastMessageSentAt = System.nanoTime();
349 this.sessionState.messageSent(msg);
354 this.channel.flush();
357 @SuppressWarnings("checkstyle:illegalCatch")
358 synchronized void write(final Notification msg) {
360 writeEpilogue(this.channel.write(msg), msg);
361 } catch (final Exception e) {
362 LOG.warn("Message {} was not sent.", msg, e);
366 synchronized ChannelFuture writeAndFlush(final Notification msg) {
367 if (this.channel.isWritable()) {
368 return writeEpilogue(this.channel.writeAndFlush(msg), msg);
370 return this.channel.newFailedFuture(new NonWritableChannelException());
374 public synchronized void closeWithoutMessage() {
375 if (this.state == State.IDLE) {
378 LOG.info("Closing session: {}", this);
379 this.channel.close().addListener((ChannelFutureListener) future -> {
380 if (future.isSuccess()) {
381 LOG.debug("Channel {} closed successfully", this.channel);
383 LOG.warn("Channel {} failed to close", this.channel, future.cause());
387 this.state = State.IDLE;
389 this.sessionState.setSessionState(this.state);
393 * Closes BGP session from the parent with given reason. A message needs to be sent, but parent doesn't have to be
394 * modified, because he initiated the closing. (To prevent concurrent modification exception).
396 * @param cause BGPDocumentedException
399 @Holding({"this.listener", "this"})
400 void terminate(final BGPDocumentedException cause) {
401 final BGPError error = cause.getError();
402 final byte[] data = cause.getData();
403 final NotifyBuilder builder = new NotifyBuilder().setErrorCode(error.getCode())
404 .setErrorSubcode(error.getSubcode());
405 if (data != null && data.length != 0) {
406 builder.setData(data);
408 writeAndFlush(builder.build());
409 notifyTerminationReasonAndCloseWithoutMessage(error);
412 private void removePeerSession() {
413 if (this.peerRegistry != null) {
414 this.peerRegistry.removePeerSession(StrictBGPPeerRegistry.getIpAddress(this.channel.remoteAddress()));
419 * If HoldTimer expires, the session ends. If a message (whichever) was received during this period, the HoldTimer
420 * will be rescheduled by HOLD_TIMER_VALUE + the time that has passed from the start of the HoldTimer to the time at
421 * which the message was received. If the session was closed by the time this method starts to execute (the session
422 * state will become IDLE), then rescheduling won't occur.
424 private void handleHoldTimer() {
425 // synchronize on listener and then on this object to ensure correct order of locking
426 synchronized (this.listener) {
427 synchronized (this) {
428 if (this.state == State.IDLE) {
432 final long ct = System.nanoTime();
433 final long nextHold = this.lastMessageReceivedAt + holdTimerNanos;
435 if (ct >= nextHold) {
436 LOG.debug("HoldTimer expired. {}", new Date());
437 terminate(new BGPDocumentedException(BGPError.HOLD_TIMER_EXPIRED));
439 this.channel.eventLoop().schedule(this::handleHoldTimer, nextHold - ct, TimeUnit.NANOSECONDS);
446 * If KeepAlive Timer expires, sends KeepAlive message. If a message (whichever) was send during this period, the
447 * KeepAlive Timer will be rescheduled by KEEP_ALIVE_TIMER_VALUE + the time that has passed from the start of the
448 * KeepAlive timer to the time at which the message was sent. If the session was closed by the time this method
449 * starts to execute (the session state will become IDLE), that rescheduling won't occur.
451 private synchronized void handleKeepaliveTimer() {
452 if (this.state == State.IDLE) {
453 LOG.debug("Skipping keepalive on session idle {}", this);
457 final long ct = System.nanoTime();
458 final long nextKeepalive = this.lastMessageSentAt + keepAliveNanos;
459 long nextNanos = nextKeepalive - ct;
461 if (nextNanos <= 0) {
462 final ChannelFuture future = this.writeAndFlush(KEEP_ALIVE);
463 LOG.debug("Enqueued session {} keepalive as {}", this, future);
464 nextNanos = keepAliveNanos;
465 if (LOG.isDebugEnabled()) {
466 future.addListener(compl -> LOG.debug("Session {} keepalive completed as {}", this, compl));
469 LOG.debug("Skipping keepalive on session {}", this);
472 LOG.debug("Scheduling next keepalive on {} in {} nanos", this, nextNanos);
473 this.channel.eventLoop().schedule(this::handleKeepaliveTimer, nextNanos, TimeUnit.NANOSECONDS);
477 public final String toString() {
478 return addToStringAttributes(MoreObjects.toStringHelper(this)).toString();
481 protected ToStringHelper addToStringAttributes(final ToStringHelper toStringHelper) {
482 toStringHelper.add("channel", this.channel);
483 toStringHelper.add("state", this.getState());
484 return toStringHelper;
488 public Set<BgpTableType> getAdvertisedTableTypes() {
489 return this.tableTypes;
493 public List<AddressFamilies> getAdvertisedAddPathTableTypes() {
494 return this.addPathTypes;
498 public GracefulRestartCapability getAdvertisedGracefulRestartCapability() {
499 return this.gracefulCapability;
503 public LlGracefulRestartCapability getAdvertisedLlGracefulRestartCapability() {
504 return this.llGracefulCapability;
508 @SuppressWarnings("checkstyle:illegalCatch")
510 // synchronize on listener and then on this object to ensure correct order of locking
511 synchronized (this.listener) {
512 synchronized (this) {
513 this.state = State.UP;
515 this.sessionState.setSessionState(this.state);
516 this.listener.onSessionUp(this);
517 } catch (final Exception e) {
525 public synchronized State getState() {
530 public final Ipv4Address getBgpId() {
535 public final AsNumber getAsNumber() {
536 return this.asNumber;
539 public ChannelOutputLimiter getLimiter() {
544 public final void channelInactive(final ChannelHandlerContext ctx) throws Exception {
545 LOG.debug("Channel {} inactive.", ctx.channel());
547 super.channelInactive(ctx);
551 protected final void channelRead0(final ChannelHandlerContext ctx, final Notification msg) {
552 LOG.trace("Message was received: {}", msg);
553 this.handleMessage(msg);
557 public final void handlerAdded(final ChannelHandlerContext ctx) {
562 public void exceptionCaught(final ChannelHandlerContext ctx, final Throwable cause) {
563 // synchronize on listener and then on this object to ensure correct order of locking
564 synchronized (this.listener) {
565 synchronized (this) {
566 handleException(cause);
572 * Handle exception occurred in the BGP session. The session in error state should be closed
573 * properly so that it can be restored later.
575 @Holding({"this.listener", "this"})
577 void handleException(final Throwable cause) {
578 LOG.warn("BGP session encountered error", cause);
579 final Throwable docCause = cause.getCause();
580 terminate(docCause instanceof BGPDocumentedException
581 ? (BGPDocumentedException) docCause : new BGPDocumentedException(BGPError.CEASE));
585 public BGPSessionState getBGPSessionState() {
586 return this.sessionState;
590 public BGPTimersState getBGPTimersState() {
591 return this.sessionState;
595 public BGPTransportState getBGPTransportState() {
596 return this.sessionState;
600 public void registerMessagesCounter(final BGPMessagesListener bgpMessagesListener) {
601 this.sessionState.registerMessagesCounter(bgpMessagesListener);
605 public <T extends PeerConstraint> void addDecoderConstraint(final Class<T> constraintClass, final T constraint) {
606 this.channel.pipeline().get(BGPByteToMessageDecoder.class).addDecoderConstraint(constraintClass, constraint);
610 public ScheduledFuture<?> schedule(final Runnable command, final long delay, final TimeUnit unit) {
611 return this.channel.eventLoop().schedule(command, delay, unit);