netconf/sal-netconf-connector/src/test/java/org/opendaylight/netconf/sal/connect/netconf/listener/NetconfDeviceCommunicatorTest.java

   1 /*
   2  * Copyright (c) 2014 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netconf.sal.connect.netconf.listener;
  10
  11 import static org.junit.Assert.assertEquals;
  12 import static org.junit.Assert.assertFalse;
  13 import static org.junit.Assert.assertNotNull;
  14 import static org.mockito.Matchers.any;
  15 import static org.mockito.Matchers.eq;
  16 import static org.mockito.Matchers.same;
  17 import static org.mockito.Mockito.doNothing;
  18 import static org.mockito.Mockito.doReturn;
  19 import static org.mockito.Mockito.mock;
  20 import static org.mockito.Mockito.never;
  21 import static org.mockito.Mockito.reset;
  22 import static org.mockito.Mockito.spy;
  23 import static org.mockito.Mockito.timeout;
  24 import static org.mockito.Mockito.verify;
  25 import static org.opendaylight.netconf.api.xml.XmlNetconfConstants.URN_IETF_PARAMS_XML_NS_NETCONF_BASE_1_0;
  26
  27 import com.google.common.base.CharMatcher;
  28 import com.google.common.base.Strings;
  29 import com.google.common.collect.Sets;
  30 import com.google.common.util.concurrent.ListenableFuture;
  31 import io.netty.channel.ChannelFuture;
  32 import io.netty.channel.EventLoopGroup;
  33 import io.netty.channel.nio.NioEventLoopGroup;
  34 import io.netty.util.HashedWheelTimer;
  35 import io.netty.util.Timer;
  36 import io.netty.util.concurrent.Future;
  37 import io.netty.util.concurrent.GenericFutureListener;
  38 import io.netty.util.concurrent.GlobalEventExecutor;
  39 import java.io.ByteArrayInputStream;
  40 import java.net.InetSocketAddress;
  41 import java.util.ArrayList;
  42 import java.util.Collection;
  43 import java.util.Collections;
  44 import java.util.UUID;
  45 import java.util.concurrent.TimeUnit;
  46 import java.util.concurrent.TimeoutException;
  47 import javax.xml.parsers.ParserConfigurationException;
  48 import org.junit.Before;
  49 import org.junit.Test;
  50 import org.mockito.ArgumentCaptor;
  51 import org.mockito.Mock;
  52 import org.mockito.MockitoAnnotations;
  53 import org.opendaylight.controller.config.util.xml.XmlMappingConstants;
  54 import org.opendaylight.netconf.api.NetconfMessage;
  55 import org.opendaylight.netconf.api.NetconfTerminationReason;
  56 import org.opendaylight.netconf.client.NetconfClientDispatcherImpl;
  57 import org.opendaylight.netconf.client.NetconfClientSession;
  58 import org.opendaylight.netconf.client.conf.NetconfClientConfiguration;
  59 import org.opendaylight.netconf.client.conf.NetconfReconnectingClientConfiguration;
  60 import org.opendaylight.netconf.client.conf.NetconfReconnectingClientConfigurationBuilder;
  61 import org.opendaylight.netconf.nettyutil.handler.ssh.authentication.LoginPassword;
  62 import org.opendaylight.netconf.sal.connect.api.RemoteDevice;
  63 import org.opendaylight.netconf.sal.connect.netconf.util.NetconfMessageTransformUtil;
  64 import org.opendaylight.netconf.sal.connect.util.RemoteDeviceId;
  65 import org.opendaylight.protocol.framework.ReconnectStrategy;
  66 import org.opendaylight.protocol.framework.TimedReconnectStrategy;
  67 import org.opendaylight.yangtools.util.xml.UntrustedXML;
  68 import org.opendaylight.yangtools.yang.common.QName;
  69 import org.opendaylight.yangtools.yang.common.RpcError;
  70 import org.opendaylight.yangtools.yang.common.RpcResult;
  71 import org.slf4j.Logger;
  72 import org.slf4j.LoggerFactory;
  73 import org.w3c.dom.Document;
  74 import org.w3c.dom.Element;
  75
  76 public class NetconfDeviceCommunicatorTest {
  77
  78     private static final Logger LOG = LoggerFactory.getLogger(NetconfDeviceCommunicatorTest.class);
  79
  80     @Mock
  81     NetconfClientSession mockSession;
  82
  83     @Mock
  84     RemoteDevice<NetconfSessionPreferences, NetconfMessage, NetconfDeviceCommunicator> mockDevice;
  85
  86     NetconfDeviceCommunicator communicator;
  87
  88     @Before
  89     public void setUp() throws Exception {
  90         MockitoAnnotations.initMocks(this);
  91
  92         communicator = new NetconfDeviceCommunicator(
  93                 new RemoteDeviceId("test", InetSocketAddress.createUnresolved("localhost", 22)), mockDevice, 10);
  94     }
  95
  96     void setupSession() {
  97         doReturn(Collections.<String>emptySet()).when(mockSession).getServerCapabilities();
  98         doNothing().when(mockDevice).onRemoteSessionUp(any(NetconfSessionPreferences.class),
  99                 any(NetconfDeviceCommunicator.class));
 100         communicator.onSessionUp(mockSession);
 101     }
 102
 103     private ListenableFuture<RpcResult<NetconfMessage>> sendRequest() throws Exception {
 104         return sendRequest(UUID.randomUUID().toString(), true);
 105     }
 106
 107     @SuppressWarnings("unchecked")
 108     private ListenableFuture<RpcResult<NetconfMessage>> sendRequest(final String messageID,
 109                                                                     final boolean doLastTest) throws Exception {
 110         Document doc = UntrustedXML.newDocumentBuilder().newDocument();
 111         Element element = doc.createElement("request");
 112         element.setAttribute("message-id", messageID);
 113         doc.appendChild(element);
 114         NetconfMessage message = new NetconfMessage(doc);
 115
 116         ChannelFuture mockChannelFuture = mock(ChannelFuture.class);
 117         doReturn(mockChannelFuture).when(mockChannelFuture)
 118                 .addListener(any((GenericFutureListener.class)));
 119         doReturn(mockChannelFuture).when(mockSession).sendMessage(same(message));
 120
 121         ListenableFuture<RpcResult<NetconfMessage>> resultFuture =
 122                 communicator.sendRequest(message, QName.create("mock rpc"));
 123         if (doLastTest) {
 124             assertNotNull("ListenableFuture is null", resultFuture);
 125         }
 126         return resultFuture;
 127     }
 128
 129     @Test
 130     public void testOnSessionUp() {
 131         String testCapability = "urn:opendaylight:params:xml:ns:test?module=test-module&revision=2014-06-02";
 132         Collection<String> serverCapabilities =
 133                 Sets.newHashSet(NetconfMessageTransformUtil.NETCONF_ROLLBACK_ON_ERROR_URI.toString(),
 134                         NetconfMessageTransformUtil.IETF_NETCONF_MONITORING.getNamespace().toString(),
 135                         testCapability);
 136         doReturn(serverCapabilities).when(mockSession).getServerCapabilities();
 137
 138         ArgumentCaptor<NetconfSessionPreferences> netconfSessionPreferences =
 139                 ArgumentCaptor.forClass(NetconfSessionPreferences.class);
 140         doNothing().when(mockDevice).onRemoteSessionUp(netconfSessionPreferences.capture(), eq(communicator));
 141
 142         communicator.onSessionUp(mockSession);
 143
 144         verify(mockSession).getServerCapabilities();
 145         verify(mockDevice).onRemoteSessionUp(netconfSessionPreferences.capture(), eq(communicator));
 146
 147         NetconfSessionPreferences actualCapabilites = netconfSessionPreferences.getValue();
 148         assertEquals("containsModuleCapability", true, actualCapabilites.containsNonModuleCapability(
 149                 NetconfMessageTransformUtil.NETCONF_ROLLBACK_ON_ERROR_URI.toString()));
 150         assertEquals("containsModuleCapability", false, actualCapabilites.containsNonModuleCapability(testCapability));
 151         assertEquals("getModuleBasedCaps", Sets.newHashSet(
 152                 QName.create("urn:opendaylight:params:xml:ns:test", "2014-06-02", "test-module")),
 153                 actualCapabilites.getModuleBasedCaps());
 154         assertEquals("isRollbackSupported", true, actualCapabilites.isRollbackSupported());
 155         assertEquals("isMonitoringSupported", true, actualCapabilites.isMonitoringSupported());
 156     }
 157
 158     @SuppressWarnings("unchecked")
 159     @Test(timeout = 5000)
 160     public void testOnSessionDown() throws Exception {
 161         setupSession();
 162
 163         ListenableFuture<RpcResult<NetconfMessage>> resultFuture1 = sendRequest();
 164         final ListenableFuture<RpcResult<NetconfMessage>> resultFuture2 = sendRequest();
 165
 166         doNothing().when(mockDevice).onRemoteSessionDown();
 167
 168         communicator.onSessionDown(mockSession, new Exception("mock ex"));
 169
 170         verifyErrorRpcResult(resultFuture1.get(), RpcError.ErrorType.TRANSPORT, "operation-failed");
 171         verifyErrorRpcResult(resultFuture2.get(), RpcError.ErrorType.TRANSPORT, "operation-failed");
 172
 173         verify(mockDevice).onRemoteSessionDown();
 174
 175         reset(mockDevice);
 176
 177         communicator.onSessionDown(mockSession, new Exception("mock ex"));
 178
 179         verify(mockDevice, never()).onRemoteSessionDown();
 180     }
 181
 182     @Test
 183     public void testOnSessionTerminated() throws Exception {
 184         setupSession();
 185
 186         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = sendRequest();
 187
 188         doNothing().when(mockDevice).onRemoteSessionDown();
 189
 190         String reasonText = "testing terminate";
 191         NetconfTerminationReason reason = new NetconfTerminationReason(reasonText);
 192         communicator.onSessionTerminated(mockSession, reason);
 193
 194         RpcError rpcError = verifyErrorRpcResult(resultFuture.get(), RpcError.ErrorType.TRANSPORT,
 195                 "operation-failed");
 196         assertEquals("RpcError message", reasonText, rpcError.getMessage());
 197
 198         verify(mockDevice).onRemoteSessionDown();
 199     }
 200
 201     @Test
 202     public void testClose() throws Exception {
 203         communicator.close();
 204         verify(mockDevice, never()).onRemoteSessionDown();
 205     }
 206
 207     @SuppressWarnings({"rawtypes", "unchecked"})
 208     @Test
 209     public void testSendRequest() throws Exception {
 210         setupSession();
 211
 212         NetconfMessage message = new NetconfMessage(UntrustedXML.newDocumentBuilder().newDocument());
 213         QName rpc = QName.create("mock rpc");
 214
 215         ArgumentCaptor<GenericFutureListener> futureListener =
 216                 ArgumentCaptor.forClass(GenericFutureListener.class);
 217
 218         ChannelFuture mockChannelFuture = mock(ChannelFuture.class);
 219         doReturn(mockChannelFuture).when(mockChannelFuture).addListener(futureListener.capture());
 220         doReturn(mockChannelFuture).when(mockSession).sendMessage(same(message));
 221
 222         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = communicator.sendRequest(message, rpc);
 223
 224         verify(mockSession).sendMessage(same(message));
 225
 226         assertNotNull("ListenableFuture is null", resultFuture);
 227
 228         verify(mockChannelFuture).addListener(futureListener.capture());
 229         Future<Void> operationFuture = mock(Future.class);
 230         doReturn(true).when(operationFuture).isSuccess();
 231         doReturn(true).when(operationFuture).isDone();
 232         futureListener.getValue().operationComplete(operationFuture);
 233
 234         try {
 235             resultFuture.get(1, TimeUnit.MILLISECONDS); // verify it's not cancelled or has an error set
 236         } catch (TimeoutException e) {
 237             LOG.info("Operation failed due timeout.");
 238         } // expected
 239     }
 240
 241     @Test
 242     public void testSendRequestWithNoSession() throws Exception {
 243         NetconfMessage message = new NetconfMessage(UntrustedXML.newDocumentBuilder().newDocument());
 244         QName rpc = QName.create("mock rpc");
 245
 246         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = communicator.sendRequest(message, rpc);
 247
 248         assertNotNull("ListenableFuture is null", resultFuture);
 249
 250         // Should have an immediate result
 251         RpcResult<NetconfMessage> rpcResult = resultFuture.get(3, TimeUnit.MILLISECONDS);
 252
 253         verifyErrorRpcResult(rpcResult, RpcError.ErrorType.TRANSPORT, "operation-failed");
 254     }
 255
 256     private static NetconfMessage createSuccessResponseMessage(final String messageID)
 257             throws ParserConfigurationException {
 258         Document doc = UntrustedXML.newDocumentBuilder().newDocument();
 259         Element rpcReply =
 260                 doc.createElementNS(URN_IETF_PARAMS_XML_NS_NETCONF_BASE_1_0, XmlMappingConstants.RPC_REPLY_KEY);
 261         rpcReply.setAttribute("message-id", messageID);
 262         Element element = doc.createElementNS("ns", "data");
 263         element.setTextContent(messageID);
 264         rpcReply.appendChild(element);
 265         doc.appendChild(rpcReply);
 266
 267         return new NetconfMessage(doc);
 268     }
 269
 270     @SuppressWarnings({ "rawtypes", "unchecked" })
 271     @Test
 272     public void testSendRequestWithWithSendFailure() throws Exception {
 273         setupSession();
 274
 275         NetconfMessage message = new NetconfMessage(UntrustedXML.newDocumentBuilder().newDocument());
 276         QName rpc = QName.create("mock rpc");
 277
 278         ArgumentCaptor<GenericFutureListener> futureListener =
 279                 ArgumentCaptor.forClass(GenericFutureListener.class);
 280
 281         ChannelFuture mockChannelFuture = mock(ChannelFuture.class);
 282         doReturn(mockChannelFuture).when(mockChannelFuture).addListener(futureListener.capture());
 283         doReturn(mockChannelFuture).when(mockSession).sendMessage(same(message));
 284
 285         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = communicator.sendRequest(message, rpc);
 286
 287         assertNotNull("ListenableFuture is null", resultFuture);
 288
 289         verify(mockChannelFuture).addListener(futureListener.capture());
 290
 291         Future<Void> operationFuture = mock(Future.class);
 292         doReturn(false).when(operationFuture).isSuccess();
 293         doReturn(true).when(operationFuture).isDone();
 294         doReturn(new Exception("mock error")).when(operationFuture).cause();
 295         futureListener.getValue().operationComplete(operationFuture);
 296
 297         // Should have an immediate result
 298         RpcResult<NetconfMessage> rpcResult = resultFuture.get(3, TimeUnit.MILLISECONDS);
 299
 300         RpcError rpcError = verifyErrorRpcResult(rpcResult, RpcError.ErrorType.TRANSPORT, "operation-failed");
 301         assertEquals("RpcError message contains \"mock error\"", true,
 302                 rpcError.getMessage().contains("mock error"));
 303     }
 304
 305     //Test scenario verifying whether missing message is handled
 306     @Test
 307     public void testOnMissingResponseMessage() throws Exception {
 308
 309         setupSession();
 310
 311         String messageID1 = UUID.randomUUID().toString();
 312         ListenableFuture<RpcResult<NetconfMessage>> resultFuture1 = sendRequest(messageID1, true);
 313
 314         String messageID2 = UUID.randomUUID().toString();
 315         ListenableFuture<RpcResult<NetconfMessage>> resultFuture2 = sendRequest(messageID2, true);
 316
 317         String messageID3 = UUID.randomUUID().toString();
 318         ListenableFuture<RpcResult<NetconfMessage>> resultFuture3 = sendRequest(messageID3, true);
 319
 320         //response messages 1,2 are omitted
 321         communicator.onMessage(mockSession, createSuccessResponseMessage(messageID3));
 322
 323         verifyResponseMessage(resultFuture3.get(), messageID3);
 324     }
 325
 326     @Test
 327     public void testOnSuccessfulResponseMessage() throws Exception {
 328         setupSession();
 329
 330         String messageID1 = UUID.randomUUID().toString();
 331         ListenableFuture<RpcResult<NetconfMessage>> resultFuture1 = sendRequest(messageID1, true);
 332
 333         String messageID2 = UUID.randomUUID().toString();
 334         final ListenableFuture<RpcResult<NetconfMessage>> resultFuture2 = sendRequest(messageID2, true);
 335
 336         communicator.onMessage(mockSession, createSuccessResponseMessage(messageID1));
 337         communicator.onMessage(mockSession, createSuccessResponseMessage(messageID2));
 338
 339         verifyResponseMessage(resultFuture1.get(), messageID1);
 340         verifyResponseMessage(resultFuture2.get(), messageID2);
 341     }
 342
 343     @Test
 344     public void testOnResponseMessageWithError() throws Exception {
 345         setupSession();
 346
 347         String messageID = UUID.randomUUID().toString();
 348         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = sendRequest(messageID, true);
 349
 350         communicator.onMessage(mockSession, createErrorResponseMessage(messageID));
 351
 352         RpcError rpcError = verifyErrorRpcResult(resultFuture.get(), RpcError.ErrorType.RPC,
 353                 "missing-attribute");
 354         assertEquals("RpcError message", "Missing attribute", rpcError.getMessage());
 355
 356         String errorInfo = rpcError.getInfo();
 357         assertNotNull("RpcError info is null", errorInfo);
 358         assertEquals("Error info contains \"foo\"", true,
 359                 errorInfo.contains("<bad-attribute>foo</bad-attribute>"));
 360         assertEquals("Error info contains \"bar\"", true,
 361                 errorInfo.contains("<bad-element>bar</bad-element>"));
 362     }
 363
 364     /**
 365      * Test whether reconnect is scheduled properly.
 366      */
 367     @Test
 368     public void testNetconfDeviceReconnectInCommunicator() throws Exception {
 369         final RemoteDevice<NetconfSessionPreferences, NetconfMessage, NetconfDeviceCommunicator> device =
 370                 mock(RemoteDevice.class);
 371
 372         final TimedReconnectStrategy timedReconnectStrategy =
 373                 new TimedReconnectStrategy(GlobalEventExecutor.INSTANCE, 10000, 0, 1.0, null, 100L, null);
 374         final ReconnectStrategy reconnectStrategy = spy(new ReconnectStrategy() {
 375             @Override
 376             public int getConnectTimeout() throws Exception {
 377                 return timedReconnectStrategy.getConnectTimeout();
 378             }
 379
 380             @Override
 381             public Future<Void> scheduleReconnect(final Throwable cause) {
 382                 return timedReconnectStrategy.scheduleReconnect(cause);
 383             }
 384
 385             @Override
 386             public void reconnectSuccessful() {
 387                 timedReconnectStrategy.reconnectSuccessful();
 388             }
 389         });
 390
 391         final EventLoopGroup group = new NioEventLoopGroup();
 392         final Timer time = new HashedWheelTimer();
 393         try {
 394             final NetconfDeviceCommunicator listener = new NetconfDeviceCommunicator(
 395                     new RemoteDeviceId("test", InetSocketAddress.createUnresolved("localhost", 22)), device, 10);
 396             final NetconfReconnectingClientConfiguration cfg = NetconfReconnectingClientConfigurationBuilder.create()
 397                     .withAddress(new InetSocketAddress("localhost", 65000))
 398                     .withReconnectStrategy(reconnectStrategy)
 399                     .withConnectStrategyFactory(() -> reconnectStrategy)
 400                     .withAuthHandler(new LoginPassword("admin", "admin"))
 401                     .withConnectionTimeoutMillis(10000)
 402                     .withProtocol(NetconfClientConfiguration.NetconfClientProtocol.SSH)
 403                     .withSessionListener(listener)
 404                     .build();
 405
 406             listener.initializeRemoteConnection(new NetconfClientDispatcherImpl(group, group, time), cfg);
 407
 408             verify(reconnectStrategy,
 409                     timeout((int) TimeUnit.MINUTES.toMillis(3)).times(101)).scheduleReconnect(any(Throwable.class));
 410         } finally {
 411             time.stop();
 412             group.shutdownGracefully();
 413         }
 414     }
 415
 416     @Test
 417     public void testOnResponseMessageWithWrongMessageID() throws Exception {
 418         setupSession();
 419
 420         String messageID = UUID.randomUUID().toString();
 421         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = sendRequest(messageID, true);
 422
 423         communicator.onMessage(mockSession, createSuccessResponseMessage(UUID.randomUUID().toString()));
 424
 425         RpcError rpcError = verifyErrorRpcResult(resultFuture.get(), RpcError.ErrorType.PROTOCOL,
 426                 "bad-attribute");
 427         assertEquals("RpcError message non-empty", true,
 428                 !Strings.isNullOrEmpty(rpcError.getMessage()));
 429
 430         String errorInfo = rpcError.getInfo();
 431         assertNotNull("RpcError info is null", errorInfo);
 432         assertEquals("Error info contains \"actual-message-id\"", true,
 433                 errorInfo.contains("actual-message-id"));
 434         assertEquals("Error info contains \"expected-message-id\"", true,
 435                 errorInfo.contains("expected-message-id"));
 436     }
 437
 438     @Test
 439     public void testConcurrentMessageLimit() throws Exception {
 440         setupSession();
 441         ArrayList<String> messageID = new ArrayList<>();
 442
 443         for (int i = 0; i < 10; i++) {
 444             messageID.add(UUID.randomUUID().toString());
 445             ListenableFuture<RpcResult<NetconfMessage>> resultFuture = sendRequest(messageID.get(i), false);
 446             assertEquals("ListenableFuture is null", true, resultFuture instanceof UncancellableFuture);
 447         }
 448
 449         final String notWorkingMessageID = UUID.randomUUID().toString();
 450         ListenableFuture<RpcResult<NetconfMessage>> resultFuture = sendRequest(notWorkingMessageID, false);
 451         assertEquals("ListenableFuture is null", false, resultFuture instanceof UncancellableFuture);
 452
 453         communicator.onMessage(mockSession, createSuccessResponseMessage(messageID.get(0)));
 454
 455         resultFuture = sendRequest(messageID.get(0), false);
 456         assertNotNull("ListenableFuture is null", resultFuture);
 457     }
 458
 459     private static NetconfMessage createErrorResponseMessage(final String messageID) throws Exception {
 460         String xmlStr = "<rpc-reply xmlns=\"urn:ietf:params:xml:ns:netconf:base:1.0\""
 461                 + "           message-id=\"" + messageID + "\">"
 462                 + "  <rpc-error>"
 463                 + "    <error-type>rpc</error-type>"
 464                 + "    <error-tag>missing-attribute</error-tag>"
 465                 + "    <error-severity>error</error-severity>"
 466                 + "    <error-message>Missing attribute</error-message>"
 467                 + "    <error-info>"
 468                 + "      <bad-attribute>foo</bad-attribute>"
 469                 + "      <bad-element>bar</bad-element>"
 470                 + "    </error-info>"
 471                 + "  </rpc-error>"
 472                 + "</rpc-reply>";
 473
 474         ByteArrayInputStream bis = new ByteArrayInputStream(xmlStr.getBytes());
 475         Document doc = UntrustedXML.newDocumentBuilder().parse(bis);
 476         return new NetconfMessage(doc);
 477     }
 478
 479     private static void verifyResponseMessage(final RpcResult<NetconfMessage> rpcResult, final String dataText) {
 480         assertNotNull("RpcResult is null", rpcResult);
 481         assertEquals("isSuccessful", true, rpcResult.isSuccessful());
 482         NetconfMessage messageResult = rpcResult.getResult();
 483         assertNotNull("getResult", messageResult);
 484 //        List<SimpleNode<?>> nodes = messageResult.getSimpleNodesByName(
 485 //                                         QName.create( URI.create( "ns" ), null, "data" ) );
 486 //        assertNotNull( "getSimpleNodesByName", nodes );
 487 //        assertEquals( "List<SimpleNode<?>> size", 1, nodes.size() );
 488 //        assertEquals( "SimpleNode value", dataText, nodes.iterator().next().getValue() );
 489     }
 490
 491     private static RpcError verifyErrorRpcResult(final RpcResult<NetconfMessage> rpcResult,
 492                                                  final RpcError.ErrorType expErrorType, final String expErrorTag) {
 493         assertNotNull("RpcResult is null", rpcResult);
 494         assertEquals("isSuccessful", false, rpcResult.isSuccessful());
 495         assertNotNull("RpcResult errors is null", rpcResult.getErrors());
 496         assertEquals("Errors size", 1, rpcResult.getErrors().size());
 497         RpcError rpcError = rpcResult.getErrors().iterator().next();
 498         assertEquals("getErrorSeverity", RpcError.ErrorSeverity.ERROR, rpcError.getSeverity());
 499         assertEquals("getErrorType", expErrorType, rpcError.getErrorType());
 500         assertEquals("getErrorTag", expErrorTag, rpcError.getTag());
 501
 502         final String msg = rpcError.getMessage();
 503         assertNotNull("getMessage is null", msg);
 504         assertFalse("getMessage is empty", msg.isEmpty());
 505         assertFalse("getMessage is blank", CharMatcher.WHITESPACE.matchesAllOf(msg));
 506         return rpcError;
 507     }
 508 }