openflow-protocol-impl/src/main/java/org/opendaylight/openflowjava/protocol/impl/core/connection/ConnectionAdapterImpl.java

   1 /*
   2  * Copyright (c) 2013 Pantheon Technologies s.r.o. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9
  10 package org.opendaylight.openflowjava.protocol.impl.core.connection;
  11
  12 import com.google.common.base.Preconditions;
  13 import com.google.common.cache.Cache;
  14 import com.google.common.cache.CacheBuilder;
  15 import com.google.common.cache.RemovalCause;
  16 import com.google.common.cache.RemovalListener;
  17 import com.google.common.cache.RemovalNotification;
  18 import com.google.common.util.concurrent.ListenableFuture;
  19 import com.google.common.util.concurrent.SettableFuture;
  20 import io.netty.channel.Channel;
  21 import io.netty.channel.ChannelFuture;
  22 import io.netty.util.concurrent.GenericFutureListener;
  23 import java.net.InetSocketAddress;
  24 import java.util.concurrent.Future;
  25 import java.util.concurrent.RejectedExecutionException;
  26 import java.util.concurrent.TimeUnit;
  27 import org.opendaylight.openflowjava.protocol.api.connection.ConnectionReadyListener;
  28 import org.opendaylight.openflowjava.protocol.api.connection.OutboundQueueHandler;
  29 import org.opendaylight.openflowjava.protocol.api.connection.OutboundQueueHandlerRegistration;
  30 import org.opendaylight.openflowjava.statistics.CounterEventTypes;
  31 import org.opendaylight.openflowjava.statistics.StatisticsCounters;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.BarrierInput;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.BarrierOutput;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoInput;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoOutput;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoReplyInput;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoRequestMessage;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.ErrorMessage;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.ExperimenterInput;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.ExperimenterMessage;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.FlowModInput;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.FlowRemovedMessage;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetAsyncInput;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetAsyncOutput;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetConfigInput;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetConfigOutput;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetFeaturesInput;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetFeaturesOutput;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetQueueConfigInput;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetQueueConfigOutput;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GroupModInput;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.HelloInput;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.HelloMessage;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.MeterModInput;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.MultipartReplyMessage;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.MultipartRequestInput;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.OfHeader;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.OpenflowProtocolListener;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PacketInMessage;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PacketOutInput;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PortModInput;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PortStatusMessage;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.RoleRequestInput;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.RoleRequestOutput;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.SetAsyncInput;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.SetConfigInput;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.TableModInput;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.system.rev130927.DisconnectEvent;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.system.rev130927.SwitchIdleEvent;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.system.rev130927.SystemNotificationsListener;
  71 import org.opendaylight.yangtools.yang.binding.DataObject;
  72 import org.opendaylight.yangtools.yang.binding.Notification;
  73 import org.opendaylight.yangtools.yang.common.RpcResult;
  74 import org.slf4j.Logger;
  75 import org.slf4j.LoggerFactory;
  76
  77 /**
  78  * Handles messages (notifications + rpcs) and connections
  79  * @author mirehak
  80  * @author michal.polkorab
  81  */
  82 public class ConnectionAdapterImpl implements ConnectionFacade {
  83     /** after this time, RPC future response objects will be thrown away (in minutes) */
  84     public static final int RPC_RESPONSE_EXPIRATION = 1;
  85
  86     /**
  87      * Default depth of write queue, e.g. we allow these many messages
  88      * to be queued up before blocking producers.
  89      */
  90     public static final int DEFAULT_QUEUE_DEPTH = 1024;
  91
  92     private static final Logger LOG = LoggerFactory
  93             .getLogger(ConnectionAdapterImpl.class);
  94     private static final Exception QUEUE_FULL_EXCEPTION =
  95             new RejectedExecutionException("Output queue is full");
  96
  97     private static final RemovalListener<RpcResponseKey, ResponseExpectedRpcListener<?>> REMOVAL_LISTENER =
  98             new RemovalListener<RpcResponseKey, ResponseExpectedRpcListener<?>>() {
  99         @Override
 100         public void onRemoval(
 101                 final RemovalNotification<RpcResponseKey, ResponseExpectedRpcListener<?>> notification) {
 102             if (! notification.getCause().equals(RemovalCause.EXPLICIT)) {
 103                 notification.getValue().discard();
 104             }
 105         }
 106     };
 107
 108     /** expiring cache for future rpcResponses */
 109     private Cache<RpcResponseKey, ResponseExpectedRpcListener<?>> responseCache;
 110
 111     private final ChannelOutboundQueue output;
 112     private final Channel channel;
 113
 114     private ConnectionReadyListener connectionReadyListener;
 115     private OpenflowProtocolListener messageListener;
 116     private SystemNotificationsListener systemListener;
 117     private OutboundQueueManager<?> outputManager;
 118     private boolean disconnectOccured = false;
 119     private final StatisticsCounters statisticsCounters;
 120     private final InetSocketAddress address;
 121
 122     /**
 123      * default ctor
 124      * @param channel the channel to be set - used for communication
 125      * @param address client address (used only in case of UDP communication,
 126      *  as there is no need to store address over tcp (stable channel))
 127      */
 128     public ConnectionAdapterImpl(final Channel channel, final InetSocketAddress address) {
 129         responseCache = CacheBuilder.newBuilder()
 130                 .concurrencyLevel(1)
 131                 .expireAfterWrite(RPC_RESPONSE_EXPIRATION, TimeUnit.MINUTES)
 132                 .removalListener(REMOVAL_LISTENER).build();
 133         this.channel = Preconditions.checkNotNull(channel);
 134         this.output = new ChannelOutboundQueue(channel, DEFAULT_QUEUE_DEPTH, address);
 135         this.address = address;
 136         channel.pipeline().addLast(output);
 137         statisticsCounters = StatisticsCounters.getInstance();
 138         LOG.debug("ConnectionAdapter created");
 139     }
 140
 141     @Override
 142     public Future<RpcResult<BarrierOutput>> barrier(final BarrierInput input) {
 143         return sendToSwitchExpectRpcResultFuture(
 144                 input, BarrierOutput.class, "barrier-input sending failed");
 145     }
 146
 147     @Override
 148     public Future<RpcResult<EchoOutput>> echo(final EchoInput input) {
 149         return sendToSwitchExpectRpcResultFuture(
 150                 input, EchoOutput.class, "echo-input sending failed");
 151     }
 152
 153     @Override
 154     public Future<RpcResult<Void>> echoReply(final EchoReplyInput input) {
 155         return sendToSwitchFuture(input, "echo-reply sending failed");
 156     }
 157
 158     @Override
 159     public Future<RpcResult<Void>> experimenter(final ExperimenterInput input) {
 160         return sendToSwitchFuture(input, "experimenter sending failed");
 161     }
 162
 163     @Override
 164     public Future<RpcResult<Void>> flowMod(final FlowModInput input) {
 165         statisticsCounters.incrementCounter(CounterEventTypes.DS_FLOW_MODS_ENTERED);
 166         return sendToSwitchFuture(input, "flow-mod sending failed");
 167     }
 168
 169     @Override
 170     public Future<RpcResult<GetConfigOutput>> getConfig(final GetConfigInput input) {
 171         return sendToSwitchExpectRpcResultFuture(
 172                 input, GetConfigOutput.class, "get-config-input sending failed");
 173     }
 174
 175     @Override
 176     public Future<RpcResult<GetFeaturesOutput>> getFeatures(
 177             final GetFeaturesInput input) {
 178         return sendToSwitchExpectRpcResultFuture(
 179                 input, GetFeaturesOutput.class, "get-features-input sending failed");
 180     }
 181
 182     @Override
 183     public Future<RpcResult<GetQueueConfigOutput>> getQueueConfig(
 184             final GetQueueConfigInput input) {
 185         return sendToSwitchExpectRpcResultFuture(
 186                 input, GetQueueConfigOutput.class, "get-queue-config-input sending failed");
 187     }
 188
 189     @Override
 190     public Future<RpcResult<Void>> groupMod(final GroupModInput input) {
 191         return sendToSwitchFuture(input, "group-mod-input sending failed");
 192     }
 193
 194     @Override
 195     public Future<RpcResult<Void>> hello(final HelloInput input) {
 196         return sendToSwitchFuture(input, "hello-input sending failed");
 197     }
 198
 199     @Override
 200     public Future<RpcResult<Void>> meterMod(final MeterModInput input) {
 201         return sendToSwitchFuture(input, "meter-mod-input sending failed");
 202     }
 203
 204     @Override
 205     public Future<RpcResult<Void>> packetOut(final PacketOutInput input) {
 206         return sendToSwitchFuture(input, "packet-out-input sending failed");
 207     }
 208
 209     @Override
 210     public Future<RpcResult<Void>> multipartRequest(final MultipartRequestInput input) {
 211         return sendToSwitchFuture(input, "multi-part-request sending failed");
 212     }
 213
 214     @Override
 215     public Future<RpcResult<Void>> portMod(final PortModInput input) {
 216         return sendToSwitchFuture(input, "port-mod-input sending failed");
 217     }
 218
 219     @Override
 220     public Future<RpcResult<RoleRequestOutput>> roleRequest(
 221             final RoleRequestInput input) {
 222         return sendToSwitchExpectRpcResultFuture(
 223                 input, RoleRequestOutput.class, "role-request-config-input sending failed");
 224     }
 225
 226     @Override
 227     public Future<RpcResult<Void>> setConfig(final SetConfigInput input) {
 228         return sendToSwitchFuture(input, "set-config-input sending failed");
 229     }
 230
 231     @Override
 232     public Future<RpcResult<Void>> tableMod(final TableModInput input) {
 233         return sendToSwitchFuture(input, "table-mod-input sending failed");
 234     }
 235
 236     @Override
 237     public Future<RpcResult<GetAsyncOutput>> getAsync(final GetAsyncInput input) {
 238         return sendToSwitchExpectRpcResultFuture(
 239                 input, GetAsyncOutput.class, "get-async-input sending failed");
 240     }
 241
 242     @Override
 243     public Future<RpcResult<Void>> setAsync(final SetAsyncInput input) {
 244         return sendToSwitchFuture(input, "set-async-input sending failed");
 245     }
 246
 247     @Override
 248     public Future<Boolean> disconnect() {
 249         ChannelFuture disconnectResult = channel.disconnect();
 250         responseCache.invalidateAll();
 251         disconnectOccured = true;
 252
 253         return handleTransportChannelFuture(disconnectResult);
 254     }
 255
 256     @Override
 257     public boolean isAlive() {
 258         return channel.isOpen();
 259     }
 260
 261     @Override
 262     public void setMessageListener(final OpenflowProtocolListener messageListener) {
 263         this.messageListener = messageListener;
 264     }
 265
 266     @Override
 267     public void consume(final DataObject message) {
 268         LOG.debug("ConsumeIntern msg on {}", channel);
 269         if (disconnectOccured ) {
 270             return;
 271         }
 272         if (message instanceof Notification) {
 273
 274             // System events
 275             if (message instanceof DisconnectEvent) {
 276                 systemListener.onDisconnectEvent((DisconnectEvent) message);
 277                 responseCache.invalidateAll();
 278                 disconnectOccured = true;
 279             } else if (message instanceof SwitchIdleEvent) {
 280                 systemListener.onSwitchIdleEvent((SwitchIdleEvent) message);
 281                 // OpenFlow messages
 282             } else if (message instanceof EchoRequestMessage) {
 283                 messageListener.onEchoRequestMessage((EchoRequestMessage) message);
 284                 statisticsCounters.incrementCounter(CounterEventTypes.US_MESSAGE_PASS);
 285             } else if (message instanceof ErrorMessage) {
 286                 // Send only unmatched errors
 287                 if (outputManager == null || !outputManager.onMessage((OfHeader) message)) {
 288                     messageListener.onErrorMessage((ErrorMessage) message);
 289                 }
 290                 statisticsCounters.incrementCounter(CounterEventTypes.US_MESSAGE_PASS);
 291             } else if (message instanceof ExperimenterMessage) {
 292                 if (outputManager != null) {
 293                     outputManager.onMessage((OfHeader) message);
 294                 }
 295                 messageListener.onExperimenterMessage((ExperimenterMessage) message);
 296                 statisticsCounters.incrementCounter(CounterEventTypes.US_MESSAGE_PASS);
 297             } else if (message instanceof FlowRemovedMessage) {
 298                 messageListener.onFlowRemovedMessage((FlowRemovedMessage) message);
 299                 statisticsCounters.incrementCounter(CounterEventTypes.US_MESSAGE_PASS);
 300             } else if (message instanceof HelloMessage) {
 301                 LOG.info("Hello received / branch");
 302                 messageListener.onHelloMessage((HelloMessage) message);
 303                 statisticsCounters.incrementCounter(CounterEventTypes.US_MESSAGE_PASS);
 304             } else if (message instanceof MultipartReplyMessage) {
 305                 if (outputManager != null) {
 306                     outputManager.onMessage((OfHeader) message);
 307                 }
 308                 messageListener.onMultipartReplyMessage((MultipartReplyMessage) message);
 309                 statisticsCounters.incrementCounter(CounterEventTypes.US_MESSAGE_PASS);
 310             } else if (message instanceof PacketInMessage) {
 311                 messageListener.onPacketInMessage((PacketInMessage) message);
 312                 statisticsCounters.incrementCounter(CounterEventTypes.US_MESSAGE_PASS);
 313             } else if (message instanceof PortStatusMessage) {
 314                 messageListener.onPortStatusMessage((PortStatusMessage) message);
 315                 statisticsCounters.incrementCounter(CounterEventTypes.US_MESSAGE_PASS);
 316             } else {
 317                 LOG.warn("message listening not supported for type: {}", message.getClass());
 318             }
 319         } else if (message instanceof OfHeader) {
 320             LOG.debug("OFheader msg received");
 321
 322             if (outputManager == null || !outputManager.onMessage((OfHeader) message)) {
 323                 RpcResponseKey key = createRpcResponseKey((OfHeader) message);
 324                 final ResponseExpectedRpcListener<?> listener = findRpcResponse(key);
 325                 if (listener != null) {
 326                     LOG.debug("corresponding rpcFuture found");
 327                     listener.completed((OfHeader)message);
 328                     statisticsCounters.incrementCounter(CounterEventTypes.US_MESSAGE_PASS);
 329                     LOG.debug("after setting rpcFuture");
 330                     responseCache.invalidate(key);
 331                 } else {
 332                     LOG.warn("received unexpected rpc response: {}", key);
 333                 }
 334             }
 335         } else {
 336             LOG.warn("message listening not supported for type: {}", message.getClass());
 337         }
 338     }
 339
 340     private <T> ListenableFuture<RpcResult<T>> enqueueMessage(final AbstractRpcListener<T> promise) {
 341         LOG.debug("Submitting promise {}", promise);
 342
 343         if (!output.enqueue(promise)) {
 344             LOG.debug("Message queue is full, rejecting execution");
 345             promise.failedRpc(QUEUE_FULL_EXCEPTION);
 346         } else {
 347             LOG.debug("Promise enqueued successfully");
 348         }
 349
 350         return promise.getResult();
 351     }
 352
 353     /**
 354      * sends given message to switch, sending result will be reported via return value
 355      * @param input message to send
 356      * @param failureInfo describes, what type of message caused failure by sending
 357      * @return future object, <ul>
 358      *  <li>if send successful, {@link RpcResult} without errors and successful
 359      *  status will be returned, </li>
 360      *  <li>else {@link RpcResult} will contain errors and failed status</li>
 361      *  </ul>
 362      */
 363     private ListenableFuture<RpcResult<Void>> sendToSwitchFuture(
 364             final DataObject input, final String failureInfo) {
 365         statisticsCounters.incrementCounter(CounterEventTypes.DS_ENTERED_OFJAVA);
 366         return enqueueMessage(new SimpleRpcListener(input, failureInfo));
 367     }
 368
 369     /**
 370      * sends given message to switch, sending result or switch response will be reported via return value
 371      * @param input message to send
 372      * @param responseClazz type of response
 373      * @param failureInfo describes, what type of message caused failure by sending
 374      * @return future object, <ul>
 375      *  <li>if send fails, {@link RpcResult} will contain errors and failed status </li>
 376      *  <li>else {@link RpcResult} will be stored in responseCache and wait for particular timeout
 377      *  ({@link ConnectionAdapterImpl#RPC_RESPONSE_EXPIRATION}),
 378      *  <ul><li>either switch will manage to answer
 379      *  and then corresponding response message will be set into returned future</li>
 380      *  <li>or response in cache will expire and returned future will be cancelled</li></ul>
 381      *  </li>
 382      *  </ul>
 383      */
 384     private <IN extends OfHeader, OUT extends OfHeader> ListenableFuture<RpcResult<OUT>> sendToSwitchExpectRpcResultFuture(
 385             final IN input, final Class<OUT> responseClazz, final String failureInfo) {
 386         final RpcResponseKey key = new RpcResponseKey(input.getXid(), responseClazz.getName());
 387         final ResponseExpectedRpcListener<OUT> listener =
 388                 new ResponseExpectedRpcListener<>(input, failureInfo, responseCache, key);
 389         statisticsCounters.incrementCounter(CounterEventTypes.DS_ENTERED_OFJAVA);
 390         return enqueueMessage(listener);
 391     }
 392
 393     /**
 394      * @param resultFuture
 395      * @param failureInfo
 396      * @param errorSeverity
 397      * @param message
 398      * @return
 399      */
 400     private static SettableFuture<Boolean> handleTransportChannelFuture(
 401             final ChannelFuture resultFuture) {
 402
 403         final SettableFuture<Boolean> transportResult = SettableFuture.create();
 404
 405         resultFuture.addListener(new GenericFutureListener<io.netty.util.concurrent.Future<? super Void>>() {
 406
 407             @Override
 408             public void operationComplete(
 409                     final io.netty.util.concurrent.Future<? super Void> future)
 410                     throws Exception {
 411                 transportResult.set(future.isSuccess());
 412                 if (!future.isSuccess()) {
 413                     transportResult.setException(future.cause());
 414                 }
 415             }
 416         });
 417         return transportResult;
 418     }
 419
 420     /**
 421      * @param message
 422      * @return
 423      */
 424     private static RpcResponseKey createRpcResponseKey(final OfHeader message) {
 425         return new RpcResponseKey(message.getXid(), message.getImplementedInterface().getName());
 426     }
 427
 428     /**
 429      * @return
 430      */
 431     private ResponseExpectedRpcListener<?> findRpcResponse(final RpcResponseKey key) {
 432         return responseCache.getIfPresent(key);
 433     }
 434
 435     @Override
 436     public void setSystemListener(final SystemNotificationsListener systemListener) {
 437         this.systemListener = systemListener;
 438     }
 439
 440     @Override
 441     public void checkListeners() {
 442         final StringBuilder buffer =  new StringBuilder();
 443         if (systemListener == null) {
 444             buffer.append("SystemListener ");
 445         }
 446         if (messageListener == null) {
 447             buffer.append("MessageListener ");
 448         }
 449         if (connectionReadyListener == null) {
 450             buffer.append("ConnectionReadyListener ");
 451         }
 452
 453         Preconditions.checkState(buffer.length() == 0, "Missing listeners: %s", buffer.toString());
 454     }
 455
 456     @Override
 457     public void fireConnectionReadyNotification() {
 458         new Thread(new Runnable() {
 459             @Override
 460             public void run() {
 461                 connectionReadyListener.onConnectionReady();
 462             }
 463         }).start();
 464     }
 465
 466     @Override
 467     public void setConnectionReadyListener(
 468             final ConnectionReadyListener connectionReadyListener) {
 469         this.connectionReadyListener = connectionReadyListener;
 470     }
 471
 472     @Override
 473     public InetSocketAddress getRemoteAddress() {
 474         return (InetSocketAddress) channel.remoteAddress();
 475     }
 476
 477     /**
 478      * Used only for testing purposes
 479      * @param cache
 480      */
 481     public void setResponseCache(final Cache<RpcResponseKey, ResponseExpectedRpcListener<?>> cache) {
 482         this.responseCache = cache;
 483     }
 484
 485         @Override
 486     public boolean isAutoRead() {
 487         return channel.config().isAutoRead();
 488     }
 489
 490         @Override
 491     public void setAutoRead(final boolean autoRead) {
 492         channel.config().setAutoRead(autoRead);
 493     }
 494
 495     @Override
 496     public <T extends OutboundQueueHandler> OutboundQueueHandlerRegistration<T> registerOutboundQueueHandler(
 497             final T handler, final int maxQueueDepth, final long maxBarrierNanos) {
 498         Preconditions.checkState(outputManager == null, "Manager %s already registered", outputManager);
 499
 500         final OutboundQueueManager<T> ret = new OutboundQueueManager<>(this, address, handler, maxQueueDepth, maxBarrierNanos);
 501         outputManager = ret;
 502         channel.pipeline().addLast(outputManager);
 503
 504         return new OutboundQueueHandlerRegistrationImpl<T>(handler) {
 505             @Override
 506             protected void removeRegistration() {
 507                 outputManager.close();
 508                 channel.pipeline().remove(outputManager);
 509                 outputManager = null;
 510             }
 511         };
 512     }
 513
 514     Channel getChannel() {
 515         return channel;
 516     }
 517 }