openflow-protocol-impl/src/main/java/org/opendaylight/openflowjava/protocol/impl/connection/ConnectionAdapterImpl.java

   1 /* Copyright (C)2013 Pantheon Technologies, s.r.o. All rights reserved. */
   2
   3 package org.opendaylight.openflowjava.protocol.impl.connection;
   4
   5 import io.netty.channel.Channel;
   6 import io.netty.channel.ChannelFuture;
   7 import io.netty.util.concurrent.GenericFutureListener;
   8
   9 import java.util.Collection;
  10 import java.util.Collections;
  11 import java.util.List;
  12 import java.util.concurrent.ExecutorService;
  13 import java.util.concurrent.Executors;
  14 import java.util.concurrent.Future;
  15 import java.util.concurrent.TimeUnit;
  16
  17 import org.opendaylight.controller.sal.common.util.RpcErrors;
  18 import org.opendaylight.controller.sal.common.util.Rpcs;
  19 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.BarrierInput;
  20 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.BarrierOutput;
  21 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoInput;
  22 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoOutput;
  23 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoReplyInput;
  24 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.EchoRequestMessage;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.ErrorMessage;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.ExperimenterInput;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.ExperimenterMessage;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.FlowModInput;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.FlowRemovedMessage;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetAsyncInput;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetAsyncOutput;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetConfigInput;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetConfigOutput;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetFeaturesInput;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetFeaturesOutput;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetQueueConfigInput;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GetQueueConfigOutput;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.GroupModInput;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.HelloInput;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.HelloMessage;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.MeterModInput;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.MultipartReplyMessage;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.MultipartRequestMessage;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.OfHeader;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.OpenflowProtocolListener;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PacketInMessage;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PacketOutInput;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PortModInput;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.PortStatusMessage;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.RoleRequestInput;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.RoleRequestOutput;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.SetAsyncInput;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.SetConfigInput;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.protocol.rev130731.TableModInput;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.system.rev130927.DisconnectEvent;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.system.rev130927.SwitchIdleEvent;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflow.system.rev130927.SystemNotificationsListener;
  58 import org.opendaylight.yangtools.yang.binding.DataObject;
  59 import org.opendaylight.yangtools.yang.binding.Notification;
  60 import org.opendaylight.yangtools.yang.common.RpcError;
  61 import org.opendaylight.yangtools.yang.common.RpcError.ErrorSeverity;
  62 import org.opendaylight.yangtools.yang.common.RpcError.ErrorType;
  63 import org.opendaylight.yangtools.yang.common.RpcResult;
  64 import org.slf4j.Logger;
  65 import org.slf4j.LoggerFactory;
  66
  67 import com.google.common.cache.Cache;
  68 import com.google.common.cache.CacheBuilder;
  69 import com.google.common.cache.RemovalListener;
  70 import com.google.common.cache.RemovalNotification;
  71 import com.google.common.collect.Lists;
  72 import com.google.common.util.concurrent.SettableFuture;
  73
  74 /**
  75  * @author mirehak
  76  * @author michal.polkorab
  77  */
  78 public class ConnectionAdapterImpl implements ConnectionFacade {
  79
  80     /** after this time, rpc future response objects will be thrown away (in minutes) */
  81     public static final int RPC_RESPONSE_EXPIRATION = 1;
  82
  83     protected static final Logger LOG = LoggerFactory
  84             .getLogger(ConnectionAdapterImpl.class);
  85
  86     private static final String APPLICATION_TAG = "OPENFLOW_LIBRARY";
  87     private static final String TAG = "OPENFLOW";
  88     private Channel channel;
  89     private OpenflowProtocolListener messageListener;
  90     /** expiring cache for future rpcResponses */
  91     protected Cache<RpcResponseKey, SettableFuture<?>> responseCache;
  92     private SystemNotificationsListener systemListener;
  93     private boolean disconnectOccured = false;
  94     private ExecutorService threadPool;
  95
  96     /**
  97      * default ctor
  98      */
  99     public ConnectionAdapterImpl() {
 100         responseCache = CacheBuilder.newBuilder()
 101                 .concurrencyLevel(1)
 102                 .expireAfterWrite(RPC_RESPONSE_EXPIRATION, TimeUnit.MINUTES)
 103                 .removalListener(new ResponseRemovalListener()).build();
 104         threadPool = Executors.newCachedThreadPool();
 105         LOG.info("ConnectionAdapter created");
 106     }
 107
 108     /**
 109      * @param channel the channel to be set - used for communication
 110      */
 111     public void setChannel(Channel channel) {
 112         this.channel = channel;
 113     }
 114
 115     @Override
 116     public Future<RpcResult<BarrierOutput>> barrier(BarrierInput input) {
 117         return sendToSwitchExpectRpcResultFuture(
 118                 input, BarrierOutput.class, "barrier-input sending failed");
 119     }
 120
 121     @Override
 122     public Future<RpcResult<EchoOutput>> echo(EchoInput input) {
 123         return sendToSwitchExpectRpcResultFuture(
 124                 input, EchoOutput.class, "echo-input sending failed");
 125     }
 126
 127     @Override
 128     public Future<RpcResult<Void>> echoReply(EchoReplyInput input) {
 129         return sendToSwitchFuture(input, "echo-reply sending failed");
 130     }
 131
 132     @Override
 133     public Future<RpcResult<Void>> experimenter(ExperimenterInput input) {
 134         return sendToSwitchFuture(input, "experimenter sending failed");
 135     }
 136
 137     @Override
 138     public Future<RpcResult<Void>> flowMod(FlowModInput input) {
 139         return sendToSwitchFuture(input, "flow-mod sending failed");
 140     }
 141
 142     @Override
 143     public Future<RpcResult<GetConfigOutput>> getConfig(GetConfigInput input) {
 144         return sendToSwitchExpectRpcResultFuture(
 145                 input, GetConfigOutput.class, "get-config-input sending failed");
 146     }
 147
 148     @Override
 149     public Future<RpcResult<GetFeaturesOutput>> getFeatures(
 150             GetFeaturesInput input) {
 151         return sendToSwitchExpectRpcResultFuture(
 152                 input, GetFeaturesOutput.class, "get-features-input sending failed");
 153     }
 154
 155     @Override
 156     public Future<RpcResult<GetQueueConfigOutput>> getQueueConfig(
 157             GetQueueConfigInput input) {
 158         return sendToSwitchExpectRpcResultFuture(
 159                 input, GetQueueConfigOutput.class, "get-queue-config-input sending failed");
 160     }
 161
 162     @Override
 163     public Future<RpcResult<Void>> groupMod(GroupModInput input) {
 164         return sendToSwitchFuture(input, "group-mod-input sending failed");
 165     }
 166
 167     @Override
 168     public Future<RpcResult<Void>> hello(HelloInput input) {
 169         return sendToSwitchFuture(input, "hello-input sending failed");
 170     }
 171
 172     @Override
 173     public Future<RpcResult<Void>> meterMod(MeterModInput input) {
 174         return sendToSwitchFuture(input, "meter-mod-input sending failed");
 175     }
 176
 177     @Override
 178     public Future<RpcResult<Void>> packetOut(PacketOutInput input) {
 179         return sendToSwitchFuture(input, "packet-out-input sending failed");
 180     }
 181
 182     @Override
 183     public Future<RpcResult<Void>> portMod(PortModInput input) {
 184         return sendToSwitchFuture(input, "port-mod-input sending failed");
 185     }
 186
 187     @Override
 188     public Future<RpcResult<RoleRequestOutput>> roleRequest(
 189             RoleRequestInput input) {
 190         return sendToSwitchExpectRpcResultFuture(
 191                 input, RoleRequestOutput.class, "role-request-config-input sending failed");
 192     }
 193
 194     @Override
 195     public Future<RpcResult<Void>> setConfig(SetConfigInput input) {
 196         return sendToSwitchFuture(input, "set-config-input sending failed");
 197     }
 198
 199     @Override
 200     public Future<RpcResult<Void>> tableMod(TableModInput input) {
 201         return sendToSwitchFuture(input, "table-mod-input sending failed");
 202     }
 203
 204     @Override
 205     public Future<RpcResult<GetAsyncOutput>> getAsync(GetAsyncInput input) {
 206         return sendToSwitchExpectRpcResultFuture(
 207                 input, GetAsyncOutput.class, "get-async-input sending failed");
 208     }
 209
 210     @Override
 211     public Future<RpcResult<Void>> setAsync(SetAsyncInput input) {
 212         return sendToSwitchFuture(input, "set-async-input sending failed");
 213     }
 214
 215     @Override
 216     public Future<Boolean> disconnect() {
 217         ChannelFuture disconnectResult = channel.disconnect();
 218         responseCache.invalidateAll();
 219         disconnectOccured = true;
 220
 221         String failureInfo = "switch disconnecting failed";
 222         ErrorSeverity errorSeverity = ErrorSeverity.ERROR;
 223         String message = "Check the switch connection";
 224         return handleTransportChannelFuture(disconnectResult, failureInfo, errorSeverity, message);
 225     }
 226
 227     @Override
 228     public boolean isAlive() {
 229         return channel.isOpen();
 230     }
 231
 232     @Override
 233     public void setMessageListener(OpenflowProtocolListener messageListener) {
 234         this.messageListener = messageListener;
 235     }
 236
 237     @Override
 238     public void consume(final DataObject message) {
 239         threadPool.execute(new Runnable() {
 240             @Override
 241             public void run() {
 242                 consumeIntern(message);
 243             }
 244         });
 245     }
 246
 247     protected void consumeIntern(final DataObject message) {
 248         LOG.debug("Consume msg");
 249         if (disconnectOccured ) {
 250             return;
 251         }
 252         if (message instanceof Notification) {
 253             // System events
 254             if (message instanceof DisconnectEvent) {
 255                 systemListener.onDisconnectEvent((DisconnectEvent) message);
 256                 responseCache.invalidateAll();
 257                 disconnectOccured = true;
 258             } else if (message instanceof SwitchIdleEvent) {
 259                 systemListener.onSwitchIdleEvent((SwitchIdleEvent) message);
 260             }
 261             // OpenFlow messages
 262               else if (message instanceof EchoRequestMessage) {
 263                 messageListener.onEchoRequestMessage((EchoRequestMessage) message);
 264             } else if (message instanceof ErrorMessage) {
 265                 messageListener.onErrorMessage((ErrorMessage) message);
 266             } else if (message instanceof ExperimenterMessage) {
 267                 messageListener.onExperimenterMessage((ExperimenterMessage) message);
 268             } else if (message instanceof FlowRemovedMessage) {
 269                 messageListener.onFlowRemovedMessage((FlowRemovedMessage) message);
 270             } else if (message instanceof HelloMessage) {
 271                 LOG.info("Hello received / branch");
 272                 messageListener.onHelloMessage((HelloMessage) message);
 273             } else if (message instanceof MultipartReplyMessage) {
 274                 messageListener.onMultipartReplyMessage((MultipartReplyMessage) message);
 275             } else if (message instanceof MultipartRequestMessage) {
 276                 messageListener.onMultipartRequestMessage((MultipartRequestMessage) message);
 277             } else if (message instanceof PacketInMessage) {
 278                 messageListener.onPacketInMessage((PacketInMessage) message);
 279             } else if (message instanceof PortStatusMessage) {
 280                 messageListener.onPortStatusMessage((PortStatusMessage) message);
 281             } else {
 282                 LOG.warn("message listening not supported for type: "+message.getClass());
 283             }
 284         } else {
 285             if (message instanceof OfHeader) {
 286                 LOG.debug("OFheader msg received");
 287                 RpcResponseKey key = createRpcResponseKey((OfHeader) message);
 288                 final SettableFuture<RpcResult<?>> rpcFuture = findRpcResponse(key);
 289                 if (rpcFuture != null) {
 290                     LOG.debug("corresponding rpcFuture found");
 291                     List<RpcError> errors = Collections.emptyList();
 292                     LOG.debug("before setting rpcFuture");
 293                     rpcFuture.set(Rpcs.getRpcResult(true, message, errors));
 294                     LOG.debug("after setting rpcFuture");
 295                     responseCache.invalidate(key);
 296                 } else {
 297                     LOG.warn("received unexpected rpc response: "+key);
 298                 }
 299             } else {
 300                 LOG.warn("message listening not supported for type: "+message.getClass());
 301             }
 302         }
 303     }
 304
 305     /**
 306      * sends given message to switch, sending result will be reported via return value
 307      * @param input message to send
 308      * @param failureInfo describes, what type of message caused failure by sending
 309      * @return future object, <ul>
 310      *  <li>if send successful, {@link RpcResult} without errors and successful
 311      *  status will be returned, </li>
 312      *  <li>else {@link RpcResult} will contain errors and failed status</li>
 313      *  </ul>
 314      */
 315     private SettableFuture<RpcResult<Void>> sendToSwitchFuture(
 316             DataObject input, final String failureInfo) {
 317         LOG.debug("going to flush");
 318         ChannelFuture resultFuture = channel.writeAndFlush(input);
 319         LOG.debug("flushed");
 320
 321         ErrorSeverity errorSeverity = ErrorSeverity.ERROR;
 322         String errorMessage = "check switch connection";
 323         return handleRpcChannelFuture(resultFuture, failureInfo, errorSeverity, errorMessage);
 324     }
 325
 326     /**
 327      * sends given message to switch, sending result or switch response will be reported via return value
 328      * @param input message to send
 329      * @param responseClazz type of response
 330      * @param failureInfo describes, what type of message caused failure by sending
 331      * @return future object, <ul>
 332      *  <li>if send fails, {@link RpcResult} will contain errors and failed status </li>
 333      *  <li>else {@link RpcResult} will be stored in responseCache and wait for particular timeout
 334      *  ({@link ConnectionAdapterImpl#RPC_RESPONSE_EXPIRATION}),
 335      *  <ul><li>either switch will manage to answer
 336      *  and then corresponding response message will be set into returned future</li>
 337      *  <li>or response in cache will expire and returned future will be cancelled</li></ul>
 338      *  </li>
 339      *  </ul>
 340      */
 341     private <IN extends OfHeader, OUT extends OfHeader> SettableFuture<RpcResult<OUT>> sendToSwitchExpectRpcResultFuture(
 342             IN input, Class<OUT> responseClazz, final String failureInfo) {
 343         LOG.debug("going to flush");
 344         SettableFuture<RpcResult<OUT>> rpcResult = SettableFuture.create();
 345         RpcResponseKey key = new RpcResponseKey(input.getXid(), responseClazz);
 346         responseCache.put(key, rpcResult);
 347         ChannelFuture resultFuture = channel.writeAndFlush(input);
 348         LOG.debug("flushed");
 349
 350         ErrorSeverity errorSeverity = ErrorSeverity.ERROR;
 351         String errorMessage = "check switch connection";
 352
 353         return handleRpcChannelFutureWithResponse(resultFuture, failureInfo, errorSeverity,
 354                 errorMessage, input, responseClazz, rpcResult, key);
 355     }
 356
 357     /**
 358      * @param resultFuture
 359      * @param failureInfo
 360      * @return
 361      */
 362     private SettableFuture<RpcResult<Void>> handleRpcChannelFuture(
 363             ChannelFuture resultFuture, final String failureInfo,
 364             final ErrorSeverity errorSeverity, final String errorMessage) {
 365
 366         final SettableFuture<RpcResult<Void>> rpcResult = SettableFuture.create();
 367         LOG.debug("handlerpcchannelfuture");
 368         resultFuture.addListener(new GenericFutureListener<io.netty.util.concurrent.Future<? super Void>>() {
 369
 370             @Override
 371             public void operationComplete(
 372                     io.netty.util.concurrent.Future<? super Void> future)
 373                     throws Exception {
 374                 LOG.debug("operation complete");
 375                 Collection<RpcError> errors = Collections.emptyList();
 376
 377                 if (future.cause() != null) {
 378                     LOG.debug("future.cause != null");
 379                     RpcError rpcError = buildRpcError(failureInfo,
 380                             errorSeverity, errorMessage, future.cause());
 381                     errors = Lists.newArrayList(rpcError);
 382                 }
 383
 384                 rpcResult.set(Rpcs.getRpcResult(
 385                         future.isSuccess(),
 386                         (Void) null,
 387                         errors)
 388                 );
 389             }
 390         });
 391         return rpcResult;
 392     }
 393
 394     /**
 395      * @param resultFuture
 396      * @param failureInfo
 397      * @param errorSeverity
 398      * @param errorMessage
 399      * @param input
 400      * @param responseClazz
 401      * @param key TODO
 402      * @param future TODO
 403      * @return
 404      */
 405     private <IN extends OfHeader, OUT extends OfHeader> SettableFuture<RpcResult<OUT>> handleRpcChannelFutureWithResponse(
 406             ChannelFuture resultFuture, final String failureInfo,
 407             final ErrorSeverity errorSeverity, final String errorMessage,
 408             final IN input, Class<OUT> responseClazz, final SettableFuture<RpcResult<OUT>> rpcResult, final RpcResponseKey key) {
 409         LOG.debug("handleRpcchanfuture with response");
 410
 411         resultFuture.addListener(new GenericFutureListener<io.netty.util.concurrent.Future<? super Void>>() {
 412
 413             @Override
 414             public void operationComplete(
 415                     io.netty.util.concurrent.Future<? super Void> future)
 416                     throws Exception {
 417
 418                 LOG.debug("operation complete");
 419                 Collection<RpcError> errors = Collections.emptyList();
 420                 if (future.cause() != null) {
 421                     LOG.debug("ChannelFuture.cause != null");
 422                     RpcError rpcError = buildRpcError(failureInfo,
 423                             errorSeverity, errorMessage, future.cause());
 424                     errors = Lists.newArrayList(rpcError);
 425                     rpcResult.set(Rpcs.getRpcResult(
 426                             future.isSuccess(),
 427                             (OUT) null,
 428                             errors)
 429                             );
 430                     responseCache.invalidate(key);
 431                 } else {
 432                     LOG.debug("ChannelFuture.cause == null");
 433                     if (responseCache.getIfPresent(key) == null) {
 434                         LOG.debug("responcache: key wasn't present");
 435                     }
 436                 }
 437             }
 438         });
 439         return rpcResult;
 440     }
 441
 442     /**
 443      * @param resultFuture
 444      * @param failureInfo
 445      * @param errorSeverity
 446      * @param message
 447      * @return
 448      */
 449     private static SettableFuture<Boolean> handleTransportChannelFuture(
 450             ChannelFuture resultFuture, final String failureInfo,
 451             final ErrorSeverity errorSeverity, final String message) {
 452
 453         final SettableFuture<Boolean> transportResult = SettableFuture.create();
 454
 455         resultFuture.addListener(new GenericFutureListener<io.netty.util.concurrent.Future<? super Void>>() {
 456
 457             @Override
 458             public void operationComplete(
 459                     io.netty.util.concurrent.Future<? super Void> future)
 460                     throws Exception {
 461                 transportResult.set(future.isSuccess());
 462                 if (!future.isSuccess()) {
 463                     transportResult.setException(future.cause());
 464                 }
 465             }
 466         });
 467         return transportResult;
 468     }
 469
 470     /**
 471      * @param cause
 472      * @return
 473      */
 474     protected RpcError buildRpcError(String info, ErrorSeverity severity, String message,
 475             Throwable cause) {
 476         RpcError error = RpcErrors.getRpcError(APPLICATION_TAG, TAG, info, severity, message,
 477                 ErrorType.RPC, cause);
 478         return error;
 479     }
 480
 481     /**
 482      * @param cause
 483      * @return
 484      */
 485     protected RpcError buildTransportError(String info, ErrorSeverity severity, String message,
 486             Throwable cause) {
 487         RpcError error = RpcErrors.getRpcError(APPLICATION_TAG, TAG, info, severity, message,
 488                 ErrorType.TRANSPORT, cause);
 489         return error;
 490     }
 491
 492     /**
 493      * @param message
 494      * @return
 495      */
 496     private static RpcResponseKey createRpcResponseKey(OfHeader message) {
 497         return new RpcResponseKey(message.getXid(), message.getClass());
 498     }
 499
 500     /**
 501      * @return
 502      */
 503     @SuppressWarnings("unchecked")
 504     private SettableFuture<RpcResult<?>> findRpcResponse(RpcResponseKey key) {
 505         return (SettableFuture<RpcResult<?>>) responseCache.getIfPresent(key);
 506     }
 507
 508     @Override
 509     public void setSystemListener(SystemNotificationsListener systemListener) {
 510         this.systemListener = systemListener;
 511     }
 512
 513     @Override
 514     public void checkListeners() {
 515         StringBuffer buffer =  new StringBuffer();
 516         if (systemListener == null) {
 517             buffer.append("SystemListener ");
 518         }
 519         if (messageListener == null) {
 520             buffer.append("MessageListener ");
 521         }
 522
 523         if (buffer.length() > 0) {
 524             throw new IllegalStateException("Missing listeners: " + buffer.toString());
 525         }
 526     }
 527
 528     static class ResponseRemovalListener implements RemovalListener<RpcResponseKey, SettableFuture<?>> {
 529
 530         @Override
 531         public void onRemoval(
 532                 RemovalNotification<RpcResponseKey, SettableFuture<?>> notification) {
 533             SettableFuture<?> future = notification.getValue();
 534             if (!future.isDone()) {
 535                 LOG.warn("rpc response discarded: " + notification.getKey());
 536                 future.cancel(true);
 537             }
 538         }
 539     }
 540
 541 }